.gitlab-ci.yml

default:
  image: python:3.9-slim
variables:
  DOCKER_TLS_CERTDIR: ""
  APP_VERSION: "beta"
  OS_AUTH_TYPE: v3applicationcredential
  OS_AUTH_URL: https://hdf-cloud.fz-juelich.de:5000
  OS_IDENTITY_API_VERSION: 3
  OS_REGION_NAME: "HDFCloud"
  OS_INTERFACE: public
  PRODUCTION_URL: https://datacatalog.fz-juelich.de/
  PRODUCTION_DOMAIN: datacatalog.fz-juelich.de,datacatalogue.eflows4hpc.eu
  PRODUCITON_HOST: datacatalog.fz-juelich.de
  VOLUME_ID: 07a93071-5be7-4cc0-8ff3-cb34e7ed2b80
  PRODUCTION_IP: 134.94.199.59
  OLD_PROD_NAME: old-production
  PRODUCTION_NAME: datacatalog-production
  TESTING_URL: https://zam10036.zam.kfa-juelich.de/
  TESTING_DOMAIN: zam10036.zam.kfa-juelich.de
  TESTING_IP: 134.94.199.36
  OLD_TEST_NAME: old-testing
  TESTING_NAME: datacatalog-testing
  ROLLBACK_COMMIT_TAG: aa1f8345d322f2532977643043df18eb4aff3bcf # stable version on master, also tagged as 0.18 | TODO update regularily

# before script copied from gitlab docs
.before_script_template: &ssh_setup
  before_script:
    - 'command -v ssh-agent >/dev/null || ( apt-get update -y && apt-get install openssh-client gcc libxslt-dev libffi-dev libssl-dev build-essential python3-dev -y )'
    - eval $(ssh-agent -s)
    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh

stages:
  - test
  - build
  - publish
  - deploy
  - test-deployment
  - cleanup

test:
  stage: test
  script: 
    - pip install -r testing_requirements.txt
    - nosetests --with-coverage --cover-package=apiserver --cover-xml
  artifacts:
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml

light-deploy-testing:
  stage: deploy
  # only run when master is updated, unless the pipeline was triggered via the web UI
  rules:
    - if: ($CI_COMMIT_BRANCH == "master" && $MANUAL_FULL_DEPLOY !~ /true/ )
  <<: *ssh_setup
  environment: Testing
  script:
    - ssh -oStrictHostKeyChecking=accept-new apiserver@$TESTING_DOMAIN "cd /home/apiserver/datacatalog && sudo git stash && sudo git checkout master && sudo git pull --all && sudo git checkout -f $CI_COMMIT_TAG && sudo git stash clear"
    - ssh -oStrictHostKeyChecking=accept-new apiserver@$TESTING_DOMAIN "sudo /home/apiserver/datacatalog/deploy_scripts/deployment.sh /home/apiserver/datacatalog $TESTING_URL $TESTING_DOMAIN $TEST_CLIENT_ID $TEST_CLIENT_SECRET $METADATA_URL"

light-deploy-production:
  stage: deploy
  # only run when stable tag is updated, unless the pipeline was triggered via the web UI
  rules:
    - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY !~ /true/)
  tags: [stable]
  <<: *ssh_setup
  environment: Production
  script:
    - ssh -oStrictHostKeyChecking=accept-new apiserver@$PRODUCITON_HOST "cd /home/apiserver/datacatalog && sudo git stash && sudo git checkout master && sudo git pull --all && sudo git checkout -f $CI_COMMIT_TAG && sudo git stash clear"
    - ssh -oStrictHostKeyChecking=accept-new apiserver@$PRODUCITON_HOST "sudo SECRETS_ENCRYPTION_KEY=$SECRETS_ENCRYPTION_KEY /home/apiserver/datacatalog/deploy_scripts/deployment.sh /home/apiserver/datacatalog $PRODUCTION_URL $PRODUCTION_DOMAIN $CLIENT_ID $CLIENT_SECRET $METADATA_URL"

full-deploy-production:
  stage: deploy
  # only run when stable tag is assigned and the pipeline is triggered in the web UI
  rules:
     - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY == "true")
  <<: *ssh_setup
  environment: Production
  script:
    - echo "Starting the full production deployment for tag $CI_COMMIT_TAG."
    - pip install python-openstackclient
    - OLD_ID=`openstack server show $PRODUCTION_NAME -f value -c id`
    - openstack server set --name $OLD_PROD_NAME $OLD_ID
    # TODO get and locally store zip of old certificate-docker-volume
    - openstack server remove volume $OLD_ID $VOLUME_ID
    - INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor s2 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data deploy_scripts/cloudinit.yml --security-group ssh --security-group www --security-group https $PRODUCTION_NAME`
    - while [ "`openstack server show $INSTANCE_ID -c addresses -f value`" = "{}" ]; do sleep 5; done # wait until an address is available to attach the floating ip
    - openstack server add floating ip $INSTANCE_ID $PRODUCTION_IP
    - sleep 10 # ensure that next command reaches the new server, prevents host key problems
    # TODO move local zip of certificate-docker-volume to server once startup is complete
    - openstack server add volume $INSTANCE_ID $VOLUME_ID
    - sleep 20 # apparently it may take some time until the volume is available to the OS
    - ssh -oStrictHostKeyChecking=accept-new apiserver@$PRODUCTION_IP "sudo mkdir -p /app/mnt"
    - ssh -oStrictHostKeyChecking=accept-new apiserver@$PRODUCTION_IP "sudo mount /dev/vdb1 /app/mnt"
    - until ssh -oStrictHostKeyChecking=accept-new apiserver@$PRODUCTION_IP ls /finished_cloudinit >/dev/null 2>&1; do sleep 30; done # wait until cloudinit script is complete
    - ssh -oStrictHostKeyChecking=accept-new apiserver@$PRODUCTION_IP "sudo SECRETS_ENCRYPTION_KEY=$SECRETS_ENCRYPTION_KEY, /home/apiserver/datacatalog/deploy_scripts/deployment.sh /home/apiserver/datacatalog $PRODUCTION_URL $PRODUCTION_DOMAIN  $CLIENT_ID $CLIENT_SECRET $METADATA_URL"

full-deploy-testing:
  stage: deploy 
  # only run when master is updated and the pipeline is triggered in the web UI
  rules:
    - if: ($CI_COMMIT_BRANCH == "master" && $MANUAL_FULL_DEPLOY == "true")
  <<: *ssh_setup
  environment: Testing
  script:
    - echo "Starting the full testing deployment."
    - pip install python-openstackclient
    - OLD_ID=`openstack server show $TESTING_NAME -f value -c id` && server_exists=true || echo "No testing server found. It might be a first time deployment"
    - if [ "$server_exists" = true ] ; then
      openstack server set --name $OLD_TEST_NAME $OLD_ID;
      fi
    # TODO get and locally store zip of old certificate-docker-volume
    - INSTANCE_ID=`openstack server create -f value -c id --prefix IMAGE_ --flavor s1 --image 149a65b5-aeb8-499f-aaa6-ec966bd28dd6 --user-data deploy_scripts/cloudinit.yml --security-group ssh --security-group www --security-group https $TESTING_NAME`
    - while [ "`openstack server show $INSTANCE_ID -c addresses -f value`" = "{}" ]; do sleep 5; done # wait until an address is available to attach the floating ip
    - openstack server add floating ip $INSTANCE_ID $TESTING_IP
    - sleep 10 # ensure that next command reaches the new server, prevents host key problems
    # TODO move local zip of certificate-docker-volume to server once startup is complete
    - until ssh -oStrictHostKeyChecking=accept-new apiserver@$TESTING_DOMAIN ls /finished_cloudinit >/dev/null 2>&1; do sleep 30; done # wait until cloudinit script is complete
    - ssh -oStrictHostKeyChecking=accept-new apiserver@$TESTING_DOMAIN "sudo /home/apiserver/datacatalog/deploy_scripts/deployment.sh /home/apiserver/datacatalog $TESTING_URL $TESTING_DOMAIN  $TEST_CLIENT_ID $TEST_CLIENT_SECRET $METADATA_URL"

  
cleanup-failed-full-deployment:
  # check if there is an old prod or test instance, assign respective ip to it, re-attach volume, delete new instance, rename old instance
  # if there is none, this is a failed light-deployment, which is handled by another job
  # this does not guarantee a successful rollback, but unless the old instance was faulty, this should work
  stage: cleanup
  when: on_failure
  rules:
     - if: $MANUAL_FULL_DEPLOY == "true"
  <<: *ssh_setup
  script:
    - echo "This is the cleanup for the full-redeployment of the testing or production servers"
    - echo "if this job is reached, some earlier job had to have failed, this will return to the previous instance (if available)"
    - echo "A successfull cleanup can not be guaranteed, depending on the failure reason"
    - pip install python-openstackclient
    # check which old instance is present. (either old test or old production); store instance id in a var
    - OLD_TEST_ID=`openstack server show $OLD_TEST_NAME -f value -c id` && rollback_test=true || echo "No old testing server found."
    - OLD_PROD_ID=`openstack server show $OLD_PROD_NAME -f value -c id` && rollback_prod=true || echo "No old production server found."
    # if applicable: rollback test server
    - if [ "$rollback_test" = true ] ; then 
      REMOVE_ID=`openstack server show $TESTING_NAME -f value -c id` && new_deployment_exists=true || echo "No new testing server has been created.";
      openstack server set --name $TESTING_NAME $OLD_TEST_ID;
      openstack server add floating ip $OLD_TEST_ID $TESTING_IP;
      if [ "$new_deployment_exists" = true ] ; then
      openstack server delete $REMOVE_ID && echo "Deleted faulty testing server.";
      fi
      fi
    # if applicable, rollback prod server
    - if [ "$rollback_prod" = true ] ; then 
      REMOVE_ID=`openstack server show $PRODUCTION_NAME -f value -c id` || echo "No new production server found.";
      openstack server set --name $PRODUCTION_NAME $OLD_PROD_ID;
      openstack server add floating ip $OLD_PROD_ID $PRODUCTION_IP;
      openstack server remove volume $REMOVE_ID $VOLUME_ID || echo "No Volume was removed.";
      openstack server delete $REMOVE_ID && echo "Deleted faulty production server." || echo "No faulty production was deleted.";
      openstack server add volume $OLD_PROD_ID $VOLUME_ID;
      ssh -oStrictHostKeyChecking=accept-new apiserver@$PRODUCITON_HOST "sudo mount /dev/vdb1 /app/mnt";
      fi
    # gitlab should automatically alert the devs about this failure

cleanup-successful-full-deployment:
  # check if there is an old prod or test instance, and delete it if present
  stage: cleanup
  when: on_success
  rules:
     - if: $MANUAL_FULL_DEPLOY == "true"
  <<: *ssh_setup
  script:
    - echo "This is the cleanup for the full-redeployment of the testing or production servers"
    - echo "if this job is reached, all earlier jobs were successful, and any lingering old instances need to be removed"
    - pip install python-openstackclient
    - openstack server delete $OLD_TEST_NAME && echo "Deleted old testing server." || echo "No old testing server found."
    - openstack server delete $OLD_PROD_NAME && echo "Deleted old production server." || echo "No old production server found."

cleanup-failed-light-test-deployment:
  # if there is a failure with the light deployments, this tries to git checkout an earlier version and rollback to that.
  stage: cleanup
  when: on_failure
  rules:
    - if: ($CI_COMMIT_BRANCH == "master" && $MANUAL_FULL_DEPLOY !~ /true/ )
  <<: *ssh_setup
  script:
    - echo "This is the cleanup for the light-redeployment of the testing servers"
    - echo "if this job is reached, some earlier job had to have failed, this will return to a previous commit"
    - echo "A successfull cleanup can not be guaranteed, depending on the failure reason"
    - COMMIT_TAG="$ROLLBACK_COMMIT_TAG" # a stable base version here, update from time to time
    - ssh -oStrictHostKeyChecking=accept-new apiserver@$TESTING_DOMAIN "cd /home/apiserver/datacatalog && sudo git pull --all && sudo git checkout -f $COMMIT_TAG"
    - ssh -oStrictHostKeyChecking=accept-new apiserver@$TESTING_DOMAIN "sudo /home/apiserver/datacatalog/deploy_scripts/deployment.sh /home/apiserver/datacatalog $TESTING_URL $TESTING_DOMAIN"


cleanup-failed-light-production-deployment:
  # if there is a failure with the light deployments, this tries to git checkout an earlier version and rollback to that.
  stage: cleanup
  when: on_failure
  rules:
    - if: ($CI_COMMIT_TAG =~ /stable/ && $MANUAL_FULL_DEPLOY !~ /true/)
  tags: [stable]
  <<: *ssh_setup
  script:
    - echo "This is the cleanup for the light-redeployment of the production servers"
    - echo "if this job is reached, some earlier job had to have failed, this will return to a previous commit"
    - echo "A successfull cleanup can not be guaranteed, depending on the failure reason"
    - COMMIT_TAG="$ROLLBACK_COMMIT_TAG" # some stable base version here, update from time to time
    - ssh -oStrictHostKeyChecking=accept-new apiserver@$PRODUCITON_HOST "cd /home/apiserver/datacatalog && sudo git pull --all && sudo git checkout -f $COMMIT_TAG"
    - ssh -oStrictHostKeyChecking=accept-new apiserver@$PRODUCITON_HOST "sudo /home/apiserver/datacatalog/deploy_scripts/deployment.sh /home/apiserver/datacatalog $PRODUCTION_URL $PRODUCTION_DOMAIN"
  
test-testing:
  cache: {}
  stage: test-deployment 
  when: on_failure
  rules:
    - if: $CI_COMMIT_BRANCH == "master"
  script:
    - apt update && apt -y install curl
    - echo "For now, this will be a basic health check i.e. GET / and check for 2xx code."
    - echo "We are affected by the expried lets encrypt cert?"
    - 'curl --insecure -I -H "Accept: application/json" $TESTING_URL'

test-production:
  cache: {}
  stage: test-deployment
  rules:
    - if: $CI_COMMIT_TAG =~ /stable/
  tags: [stable]
  script:
    - apt update && apt -y install curl
    - echo "For now, this will be a basic health check i.e. GET / and check for 2xx code."
    - 'curl -f -H "Accept: application/json" $PRODUCTION_URL' 


publishgit-do:
  stage: publish
  only:
    - tags
  tags: [stable]
  script:
    - apt-get update 
    - apt-get install -y git
    - (git remote -v | grep gith) || git remote add gith "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/eflows4hpc/datacatalog.git"
    - git remote -v
    - git fetch --unshallow origin
    - git push gith +HEAD:refs/heads/master



# This is an automatic push of the docker image into gitLab container repository
transfer_image:
  stage: build
  image: docker:latest
  services:
    - docker:dind
  variables:
    IMAGE_COMMIT_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
    IMAGE_MASTER_TAG: $CI_REGISTRY_IMAGE:master
  script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker build --no-cache=true --pull -f ./apiserver/Dockerfile -t $IMAGE_COMMIT_TAG .
    - docker push $IMAGE_COMMIT_TAG
    - docker tag $IMAGE_COMMIT_TAG $IMAGE_MASTER_TAG
    - docker push $IMAGE_MASTER_TAG

tag_release:
  stage: publish
  image: docker:latest
  services:
    - docker:dind
  variables:
    IMAGE_LATEST_TAG: $CI_REGISTRY_IMAGE:latest
    IMAGE_STABLE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
    IMAGE_COMMIT_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
  only:
    - tags
  tags: [stable]
  script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker pull $IMAGE_COMMIT_TAG
    - docker tag $IMAGE_COMMIT_TAG $IMAGE_STABLE_TAG
    - docker tag $IMAGE_COMMIT_TAG $IMAGE_LATEST_TAG
    - docker push $IMAGE_STABLE_TAG
    - docker push $IMAGE_LATEST_TAG