.gitlab-ci.yml

image: humancellatlas/dss-build-box
# The Docker image `humancellatlas/dss-build-box` is created through a manual process from
# `${DSS_HOME}/Dockerfile.allspark`. See the contents of `${DSS_HOME}/Dockerfile.allspark`
# creation and usage instructions.

variables:
  GIT_SUBMODULE_STRATEGY: normal
  DSS_ES_TIMEOUT: 30
  DSS_UNITTEST_OPTS: "-v"
  GITHUB_API: "https://api.github.com"

stages:
  - test
  - deploy
  - integration_test
  - release

before_script:
  - export COMMITS_URL=${GITHUB_API}/repos/HumanCellAtlas/data-store/commits
  - if not [[ CI_COMMIT_SHA == $(http GET $COMMITS_URL sha==$CI_COMMIT_REF_NAME | jq -r '.[0]["sha"]') ]]; then exit 1; fi
# TODO: figure out how to get the gitlab-runner to not clone the repo as root - Brian H
  - cp -r /HumanCellAtlas/data-store ~/data-store && cd ~/data-store
  - git reset --hard HEAD
  - virtualenv ~/venv
  - source ~/venv/bin/activate
  - pip install -r requirements-dev.txt
  - source environment
  - if [[ master == $CI_COMMIT_REF_NAME ]]; then
  -   DSS_ENV="dev"
  - elif [[ -f "environment.$CI_COMMIT_REF_NAME" ]]; then
  -   DSS_ENV=$CI_COMMIT_REF_NAME
  -   source environment.$DSS_ENV
  - fi
  - scripts/fetch_secret.sh application_secrets.json > application_secrets.json
  - scripts/fetch_secret.sh gcp-credentials.json > gcp-credentials.json
  - export GOOGLE_APPLICATION_CREDENTIALS=$(pwd -P)/gcp-credentials.json

unit_tests:
  stage: test
  script:
    - whoami
    - ls -alh /
    - make -j4 parallel_test
  except:
    - tags
    - schedules

test_search:
  stage: test
  script:
    - make -j1 tests/test_search.py
  except:
    - tags
    - schedules

test_aws_indexer:
  stage: test
  variables:
    DSS_UNITTEST_OPTS: "-v TestAWSIndexer"
  script:
    - make -j1 tests/test_indexer.py
  except:
    - tags
    - schedules

test_gcp_indexer:
  stage: test
  variables:
    DSS_UNITTEST_OPTS: "-v TestGCPIndexer"
  script:
    - make -j1 tests/test_indexer.py
  except:
    - tags
    - schedules

test_subscriptions:
  stage: test
  script:
    - make -j1 tests/test_subscriptions.py
  except:
    - tags
    - schedules

test_sync_large_blob:
  stage: test
  variables:
    DSS_TEST_MODE: "integration"
  script:
    - python tests/test_sync.py TestSyncDaemon.test_sync_large_blob
  only:
    refs:
      - schedules
    variables:
      - $DSS_TEST_LARGE_FILE_SYNC

deploy:
  stage: deploy
  script:
    - make plan-infra
    - make deploy
  environment:
    name: $DSS_ENV
    url: https://dss.$DSS_ENV.data.humancellatlas.org
  only:
    - master
    - integration
    - staging
  except:
    - schedules

integration_test:
  stage: integration_test
  script:
    - make -j1 integration_test
  only:
    - master
    - integration
    - staging
  except:
    - schedules

release_integration:
  stage: release
  script:
    - git remote set-url origin https://$GITHUB_TOKEN@github.com/HumanCellAtlas/data-store.git
    - for i in $(seq 1 40); do
    -   status=$(scripts/status.sh HumanCellAtlas dcp integration)
    -   if [[ pending != "${status}" && running != "${status}" ]]; then break; fi
    -   echo "waiting for DCP Integration test to complete";
    -   sleep 30;  # This loop will check status for 20 minutes and then quit
    - done
    - if [[ success != "${status}" ]]; then
    -   echo "DCP Integration test returned status ${status}";
    -   exit 1
    - fi
    - scripts/release.sh master integration --no-deploy --skip-github-status --skip-account-verification
  only:
    - master
  except:
    - schedules
  when: manual
  allow_failure: true

force_release_integration:
  stage: release
  script:
    - git remote set-url origin https://$GITHUB_TOKEN@github.com/HumanCellAtlas/data-store.git
    - scripts/release.sh master integration --force --no-deploy --skip-github-status --skip-account-verification
  only:
    - master
  except:
    - schedules
  when: manual

release_staging:
  stage: release
  script:
    - git remote set-url origin https://$GITHUB_TOKEN@github.com/HumanCellAtlas/data-store.git
    - for i in $(seq 1 40); do
    -   status=$(scripts/status.sh HumanCellAtlas dcp staging)
    -   if [[ pending != "${status}" && running != "${status}" ]]; then break; fi
    -   echo "waiting for DCP Integration test to complete";
    -   sleep 30;  # This loop will check status for 20 minutes and then quit
    - done
    - if [[ success != "${status}" ]]; then
    -   echo "DCP Integration test returned status ${status}";
    -   exit 1
    - fi
    - scripts/release.sh integration staging --no-deploy --skip-github-status --skip-account-verification
  only:
    - integration
  except:
    - schedules
  when: manual 

force_release_staging:
  stage: release
  script:
    - git remote set-url origin https://$GITHUB_TOKEN@github.com/HumanCellAtlas/data-store.git
    - scripts/release.sh integration staging --force --no-deploy --skip-github-status --skip-account-verification
  only:
    - integration
  except:
    - schedules
  when: manual

release_prod:
  stage: release
  script:
    - git remote set-url origin https://$GITHUB_TOKEN@github.com/HumanCellAtlas/data-store.git
    - for i in $(seq 1 40); do
    -   status=$(scripts/status.sh HumanCellAtlas dcp prod)
    -   if [[ pending != "${status}" && running != "${status}" ]]; then break; fi
    -   echo "waiting for DCP Integration test to complete";
    -   sleep 30;  # This loop will check status for 20 minutes and then quit
    - done
    - if [[ success != "${status}" ]]; then
    -   echo "DCP Integration test returned status ${status}";
    -   exit 1
    - fi
    - scripts/release.sh staging prod --no-deploy --skip-github-status --skip-account-verification
  only:
    - staging
  except:
    - schedules
  when: manual 

force_release_prod:
  stage: release
  script:
    - git remote set-url origin https://$GITHUB_TOKEN@github.com/HumanCellAtlas/data-store.git
    - scripts/release.sh staging prod --force --no-deploy --skip-github-status --skip-account-verification
  only:
    - staging
  except:
    - schedules
  when: manual