.gitlab-ci.yml

variables:
  ### AWS
  AWS_REGION: us-east-1
  ### Application
  LOCAL_API_URL: http://localhost:5000/posts
  REVIEW_API_URL: https://review.devops-product-hunting.com:5000/posts
  PROD_API_URL: https://devops-product-hunting.com:5000/posts
  ### Environments
  REVIEW_URL: https://review.devops-product-hunting.com
  PROD_URL: https://devops-product-hunting.com
  ### FinOps
  TF_ROOT: infrastructure/prod
### Project CI/CD variables
# AWS_ACCESS_KEY_ID             - Protected
# AWS_ACCOUNT_ID                - Protected
# AWS_SECRET_ACCESS_KEY         - Protected/Masked
# GITLAB_TOKEN                  - Masked
# INFRACOST_API_KEY             - Masked
# PUBLIC_IP                     - Protected/Masked
# POSTGRES_DB                   - Protected
# POSTGRES_PASSWORD             - Protected/Masked
# POSTGRES_USER                 - Protected
# PRODUCT_HUNT_API_ACCESS_TOKEN - Protected/Masked
# SQLIZER_API_KEY               - Protected/Masked

stages:
  - prerequisites
  - finops
  - infrastructure
  - build
  - test
  - release
  - deploy
  - prod prerequisites
  - prod infrastructure
  - prod build
  - prod release
  - prod deploy
  - prod operate
  - prod monitor
  - prod finops
  - prod destroy

### ###
### Dev - Merge request
### ###

infracost on prod infra:
  stage: finops
  image:
    name: infracost/infracost:ci-0.10
    entrypoint: [""]
  only:
    - merge_requests
  script:
    # Clone the base branch of the pull request into a temp directory.
    - git clone $CI_REPOSITORY_URL --branch=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME --single-branch /tmp/base
    ### Generate an Infracost cost snapshot from the comparison branch, so that Infracost can compare the cost difference.
    - |
      infracost breakdown \
      --path=/tmp/base/${TF_ROOT} \
      --format=json \
      --out-file=infracost-base.json
    ### Generate an Infracost diff and save it to a JSON file.
    - |
      infracost diff \
      --path=${TF_ROOT} \
      --compare-to=infracost-base.json \
      --format=json \
      --out-file=infracost.json
    ### Post a comment to the PR using the 'update' behavior.
    - |
      infracost comment gitlab \
      --path=infracost.json \
      --repo=$CI_PROJECT_PATH \
      --merge-request=$CI_MERGE_REQUEST_IID \
      --gitlab-server-url=$CI_SERVER_URL \
      --gitlab-token=$GITLAB_TOKEN \
      --behavior=update

build client-server app:
  stage: build
  image: alpine:3.16
  only:
    - merge_requests
  cache:
    -
      key: DEV-NODE-MODULES-$CI_PIPELINE_ID
      paths:
        - $CI_PROJECT_DIR/application/backend/api/node_modules/
        - $CI_PROJECT_DIR/application/frontend/client/node_modules/
  before_script:
    - apk add --no-cache nodejs npm
  script:
    ### ### Server-side
    - cd $CI_PROJECT_DIR/application/backend/api
    ### Base dependencies
    - npm install express pg cors
    ### OpenTelemetry dependencies
    - npm install --save @opentelemetry/api @opentelemetry/sdk-node @opentelemetry/auto-instrumentations-node @opentelemetry/exporter-jaeger
    ### ### Client-side
    - cd $CI_PROJECT_DIR/application/frontend/client
    - npm install --save react-icons bootswatch

smoke test:
  stage: test
  image: alpine:3.16
  services:
    - postgres:alpine3.16
  variables:
    POSTGRES_USER: test_user
    POSTGRES_PASSWORD: test_password
    POSTGRES_DB: test_db
  only:
    - merge_requests
  cache:
    -
      key: DEV-NODE-MODULES-$CI_PIPELINE_ID
      paths:
        - $CI_PROJECT_DIR/application/backend/api/node_modules/
        - $CI_PROJECT_DIR/application/frontend/client/node_modules/
  before_script:
    - apk add --no-cache postgresql-client nodejs npm
    ### Packages for Selenium
    - apk add --no-cache python3 py3-pip chromium-chromedriver gcc python3-dev libffi-dev musl-dev
    - pip3 install selenium
  script:
    ### Push .sql file to Postgres DB
    - cd $CI_PROJECT_DIR/merge-request
    - export PGPASSWORD="$POSTGRES_PASSWORD"
    - psql -h postgres -U $POSTGRES_USER -d $POSTGRES_DB -f sample_posts.sql
    ### Launch server API
    - cd $CI_PROJECT_DIR/application/backend/api
    - export POSTGRES_HOST="postgres"
    - export POSTGRES_PORT=5432
    - node --require './tracing.js' index.js &
    ### Launch client Web application
    - cd $CI_PROJECT_DIR/application/frontend/client
    - export REACT_APP_API_URL="$LOCAL_API_URL"
    - npm start &
    ### Wait for application to start
    - sleep 15
    ### Smoke test with Selenium
    - cd $CI_PROJECT_DIR/test/smoke-test
    - python3 extractWebPage.py
    - cat page.html | grep "Rank 1"
  artifacts:
    paths:
      - test/smoke-test/page.html

include:
  - template: Code-Quality.gitlab-ci.yml
code_quality:
  ### Execute this job only during merge requests
  rules:
    - if: $CODE_QUALITY_DISABLED
      when: never
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
  variables:
    REPORT_FORMAT: html
  artifacts:
    paths: [gl-code-quality-report.html]

### ###
### Review
### ###

verify state locking:
  stage: prerequisites
  image:
    name: hashicorp/terraform:1.2.2
    entrypoint: [""]
  only:
    - main
  before_script:
    - apk add --no-cache python3 py3-pip
    - pip3 install awscli
    - aws configure set region $AWS_REGION
  script:
    ### Verify if state-lock table exists (i.e. if state lock applied)
    - aws dynamodb describe-table --table-name product-hunting-terraform-state-lock
    ### Verify is S3 state storage exists
    - aws s3 ls product-hunting-terraform-state

apply state locking:
  stage: prerequisites
  needs: ["verify state locking"]
  when: on_failure
  image:
    name: hashicorp/terraform:1.2.2
    entrypoint: [""]
  only:
    - main
  before_script:
    - apk add --no-cache python3 py3-pip
    - pip3 install awscli
    - aws configure set region $AWS_REGION
  script:
    - cd ./state-lock
    - terraform init
    - terraform apply -auto-approve

create main infra:
  stage: infrastructure
  image:
    name: hashicorp/terraform:1.2.2
    entrypoint: [""]
  only:
    - main
  before_script:
    - apk add --no-cache python3 py3-pip
    - pip3 install awscli
    - aws configure set region $AWS_REGION
  script:
    - cd ./infrastructure/main
    - terraform init
    - terraform apply -auto-approve

store top posts in sql file:
  stage: build
  image: alpine:3.16
  only:
    - main
  cache:
    -
      key: REVIEW-POSTGRES
      paths:
        - $CI_PROJECT_DIR/application/backend/worker/posts.sql
    -
      key: API-LIMITATION
      paths:
        - $CI_PROJECT_DIR/application/backend/worker/api_last_use.txt
  before_script:
    ### ### Product Hunt API limits large requests to every 15 minutes
    ### Stop if API limitation still running
    - |      
      if [ -f $CI_PROJECT_DIR/application/backend/worker/api_last_use.txt ]; then
        API_LIMIT_EXPIRE=$(($(cat $CI_PROJECT_DIR/application/backend/worker/api_last_use.txt) + 900))
        NOW=$(date "+%s")
        if [ $NOW -lt $API_LIMIT_EXPIRE ]; then
          exit 0
        fi
      fi
    ### Install dependencies
    - apk add --no-cache python3 py3-pip
    - pip3 install sqlizer-io-client
  script:
    - cd ./application/backend/worker
    ### GET top 500 most voted posts from Product Hunt API (output .json file)
    - export API_ACCESS_TOKEN=$PRODUCT_HUNT_API_ACCESS_TOKEN
    - python3 getTopPosts.py
    ### Define API last use by adding actual timestamp (needed for API limitation)
    - date '+%s' > api_last_use.txt
    ### Delete emojis inside .json file
    - sed -i -e 's/\(\\u\).\{4\}//g' posts.json
    ### Convert JSON to SQL
    - export API_KEY=$SQLIZER_API_KEY
    - python3 convertJsonToSql.py
    ### Reformat fields name & add id field
    - sed -i 's/list_node_//g' posts.sql
    - sed -i '2s/^/    "id" SERIAL PRIMARY KEY,\n/' posts.sql
    - sed -i "s/('/(DEFAULT,'/g" posts.sql

build project:
  stage: build
  image: alpine:3.16
  only:
    - main
  cache:
    -
      key: REVIEW-NODE-MODULES-$CI_PIPELINE_ID
      paths:
        - $CI_PROJECT_DIR/application/backend/api/node_modules/
    -
      key: REVIEW-BUILD-$CI_PIPELINE_ID
      paths:
        - $CI_PROJECT_DIR/application/frontend/client/build
  before_script:
    - apk add --no-cache npm
  script:
    ### ### Server-side
    - cd $CI_PROJECT_DIR/application/backend/api
    ### Base dependencies
    - npm install express pg cors
    ### OpenTelemetry dependencies
    - npm install --save @opentelemetry/api @opentelemetry/sdk-node @opentelemetry/auto-instrumentations-node @opentelemetry/exporter-jaeger
    ### ### Client-side
    - cd $CI_PROJECT_DIR/application/frontend/client
    - npm install --save react-icons bootswatch
    - export REACT_APP_API_URL="$REVIEW_API_URL"
    - npm run build

create review docker images:
  stage: release
  image: docker:20.10
  services:
    - docker:20.10-dind
  only:
    - main
  cache:
    -
      key: REVIEW-POSTGRES
      paths:
        - $CI_PROJECT_DIR/application/backend/worker/posts.sql
    -
      key: REVIEW-NODE-MODULES-$CI_PIPELINE_ID
      paths:
        - $CI_PROJECT_DIR/application/backend/api/node_modules/
    -
      key: REVIEW-BUILD-$CI_PIPELINE_ID
      paths:
        - $CI_PROJECT_DIR/application/frontend/client/build/
  before_script:
    - apk add --no-cache python3 py3-pip
    - pip3 install awscli
    - aws configure set region $AWS_REGION
  script:
    - cd ./release
    ### Build images
    - |
      docker-compose build \
      --build-arg POSTGRES_USER=$POSTGRES_USER \
      --build-arg POSTGRES_PASSWORD=$POSTGRES_PASSWORD \
      --build-arg POSTGRES_DB=$POSTGRES_DB \
      --no-cache
    ### Connect to AWS ECR registry
    - aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com
    ### Create image tags
    - docker tag product-hunting-postgres:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/product-hunting-postgres:review-$CI_COMMIT_SHORT_SHA
    - docker tag product-hunting-api:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/product-hunting-api:review-$CI_COMMIT_SHORT_SHA
    - docker tag product-hunting-client:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/product-hunting-client:review-$CI_COMMIT_SHORT_SHA
    ### Push images to ECR
    - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/product-hunting-postgres:review-$CI_COMMIT_SHORT_SHA
    - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/product-hunting-api:review-$CI_COMMIT_SHORT_SHA
    - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/product-hunting-client:review-$CI_COMMIT_SHORT_SHA

deploy review on ecs:
  stage: deploy
  image:
    name: hashicorp/terraform:1.2.2
    entrypoint: [""]
  only:
    - main
  environment:
    name: review
    url: $REVIEW_URL
    on_stop: destroy review
  before_script:
    - apk add --no-cache python3 py3-pip
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
  script:
    ### Configure Terraform
    - cd ./deploy/ecs
    - |
      cat <<EOF | tee ./main.tfvars
      ecr_registry = "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com"
      aws_account_id = "$AWS_ACCOUNT_ID"
      public_ip = "$PUBLIC_IP"
      ci_commit_short_sha = "$CI_COMMIT_SHORT_SHA"
      EOF
    - terraform init
    - terraform apply -var-file=main.tfvars -auto-approve

destroy review:
  stage: deploy
  needs: ["deploy review on ecs"]
  when: manual
  image:
    name: hashicorp/terraform:1.2.2
    entrypoint: [""]
  only:
    - main
  environment:
    name: review
    action: stop
  before_script:
    - apk add --no-cache python3 py3-pip
    - pip3 install awscli
    - aws configure set region $AWS_REGION
  script:
    - cd ./deploy/ecs
    - |
      cat <<EOF | tee ./main.tfvars
      ecr_registry = "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com"
      aws_account_id = "$AWS_ACCOUNT_ID"
      public_ip = "$PUBLIC_IP"
      ci_commit_short_sha = "$CI_COMMIT_SHORT_SHA"
      EOF
    - terraform init
    - terraform destroy -var-file=main.tfvars -auto-approve

### ###
### Production
### ###

push to prod:
  stage: prod prerequisites
  needs: ["deploy review on ecs"]
  when: manual
  variables:
    GIT_STRATEGY: none
  only:
    - main
  script:
    - ""

create prod infra:
  stage: prod infrastructure
  needs: ["push to prod"]
  when: on_success
  image:
    name: hashicorp/terraform:1.2.2
    entrypoint: [""]
  only:
    - main
  before_script:
    - apk add --no-cache python3 py3-pip
    - pip3 install awscli
    - aws configure set region $AWS_REGION
  script:
    - cd ./infrastructure/prod
    - terraform init
    - terraform apply -auto-approve

rebuild project for prod:
  stage: prod build
  needs: ["push to prod"]
  when: on_success
  image: alpine:3.16
  only:
    - main
  cache:
    -
      key: PROD-BUILD-$CI_PIPELINE_ID
      paths:
        - $CI_PROJECT_DIR/application/frontend/client/build
  before_script:
    - apk add --no-cache npm
  script:
    - cd $CI_PROJECT_DIR/application/frontend/client
    - npm install --save react-icons bootswatch
    - export REACT_APP_API_URL="$PROD_API_URL"
    - npm run build

create prod docker images:
  stage: prod release
  needs: ["rebuild project for prod"]
  when: on_success
  image: docker:20.10
  services:
    - docker:20.10-dind
  only:
    - main
  cache:
    -
      key: PROD-BUILD-$CI_PIPELINE_ID
      paths:
        - $CI_PROJECT_DIR/application/frontend/client/build/
  before_script:
    - apk add --no-cache python3 py3-pip jq
    - pip3 install awscli
    - aws configure set region $AWS_REGION
  script:
    ### Add tag to postgres image
    - MANIFEST=$(aws ecr batch-get-image --repository-name product-hunting-postgres --image-ids imageTag=review-$CI_COMMIT_SHORT_SHA --output json | jq --raw-output --join-output '.images[0].imageManifest')
    - aws ecr put-image --repository-name product-hunting-postgres --image-tag prod-$CI_COMMIT_SHORT_SHA --image-manifest "$MANIFEST"
    ### Add tag to api image
    - MANIFEST=$(aws ecr batch-get-image --repository-name product-hunting-api --image-ids imageTag=review-$CI_COMMIT_SHORT_SHA --output json | jq --raw-output --join-output '.images[0].imageManifest')
    - aws ecr put-image --repository-name product-hunting-api --image-tag prod-$CI_COMMIT_SHORT_SHA --image-manifest "$MANIFEST"
    ### Recreate client image
    - cd ./release
    - docker-compose build --no-cache client
    ### Connect to AWS ECR registry
    - aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com
    ### Create image tag
    - docker tag product-hunting-client:latest $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/product-hunting-client:prod-$CI_COMMIT_SHORT_SHA
    ### Push image to ECR
    - docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/product-hunting-client:prod-$CI_COMMIT_SHORT_SHA

deploy prod on eks:
  stage: prod deploy
  needs: ["create prod infra", "create prod docker images"]
  when: on_success
  image:
    name: hashicorp/terraform:1.2.2
    entrypoint: [""]
  only:
    - main
  environment:
    name: production
    url: $PROD_URL
    on_stop: destroy prod deploy
  before_script:
    - apk add --no-cache curl python3 py3-pip openssl
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Configure Terraform
    - cd ./deploy/eks
    - |
      cat <<EOF | tee ./main.tfvars
      ecr_registry = "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com"
      ci_commit_short_sha = "$CI_COMMIT_SHORT_SHA"
      EOF
    - terraform init
    - terraform refresh -var-file=main.tfvars
    - terraform apply -var-file=main.tfvars -auto-approve
    ### Install Kubernetes Metrics Server
    - kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

apply vertical pod autoscaler:
  stage: prod operate
  needs: ["deploy prod on eks"]
  when: on_success
  image:
    name: golang:1.18-alpine3.16
    entrypoint: [""]
  only:
    - main
  before_script:
    - apk add --no-cache curl python3 py3-pip git bash openssl
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Install Vertical Pod Autoscaler (VPA)
    - cd ./operate/vpa
    - git clone https://github.com/kubernetes/autoscaler.git
    - bash ./autoscaler/vertical-pod-autoscaler/hack/vpa-up.sh
    ### Verify VPA installation
    - kubectl get pods -n kube-system | grep vpa
    ### Create VPA object
    - kubectl apply -f product-hunting-vpa.yaml

apply horizontal pod autoscaler:
  stage: prod operate
  needs: ["deploy prod on eks"]
  when: on_success
  image:
    name: hashicorp/terraform:1.2.2
    entrypoint: [""]
  only:
    - main
  before_script:
    - apk add --no-cache curl python3 py3-pip
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Configure Terraform
    - cd ./operate/hpa
    - terraform init
    - terraform apply -auto-approve

rollback to previous revision:
  stage: prod operate
  needs: ["deploy prod on eks"]
  when: manual
  image: alpine:3.16
  only:
    - main
  before_script:
    - apk add --no-cache curl python3 py3-pip
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Perform a rollback to the previous revision
    - kubectl rollout undo deployment product-hunting

monitoring prerequisites:
  stage: prod monitor
  needs: ["deploy prod on eks"]
  when: on_success
  image: alpine:3.16
  only:
    - main
  cache:
    -
      key: PROD-PROMETHEUS-$CI_PIPELINE_ID
      paths:
        - $CI_PROJECT_DIR/monitor/kube-prometheus/default/manifests/
  before_script:
    - apk add --no-cache curl python3 py3-pip git
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### ### Needed by 'monitor kube with kube-prometheus' job
    ### Install kube-prometheus release-0.10 compatible with Kubernetes 1.22
    - cd $CI_PROJECT_DIR/monitor/kube-prometheus/default
    - git clone --depth 1 https://github.com/prometheus-operator/kube-prometheus.git -b release-0.10 /tmp/prometheus
    - cp -R /tmp/prometheus/manifests .
    - kubectl apply --server-side -f manifests/setup
    ### Add persistent storage
    - cd $CI_PROJECT_DIR/monitor/
    - kubectl apply -f persistent-storage/
    ### ### Needed by 'app tracing with jaeger' job
    ### Install cert-manager
    - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.6.3/cert-manager.yaml
    ### Install Elasticsearch Operator
    - kubectl apply -f https://download.elastic.co/downloads/eck/2.3.0/crds.yaml
    - kubectl apply -f https://download.elastic.co/downloads/eck/2.3.0/operator.yaml


monitor vertical pod autoscaler using goldilocks:
  stage: prod monitor
  needs: ["apply vertical pod autoscaler"]
  when: on_success
  image:
    name: golang:1.18-alpine3.16
    entrypoint: [""]
  only:
    - main
  before_script:
    - apk add --no-cache curl python3 py3-pip git bash
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Install Goldilocks
    - cd ./monitor/vpa
    - git clone https://github.com/FairwindsOps/goldilocks.git
    - cd goldilocks
    - kubectl create namespace goldilocks --dry-run=client -o yaml | kubectl apply -f -
    - kubectl -n goldilocks apply -f hack/manifests/controller
    - kubectl -n goldilocks apply -f hack/manifests/dashboard
    ### Enable 'default' namespace to Goldilocks Dashboard
    - kubectl label --overwrite ns default goldilocks.fairwinds.com/enabled=true
    ### ### Goldilocks is now accessible using port-forwarding
    ### kubectl -n goldilocks port-forward svc/goldilocks-dashboard 8080:80

monitor kube with kube-prometheus:
  stage: prod monitor
  needs: ["monitoring prerequisites"]
  when: delayed
  start_in: 5 minutes
  image: alpine:3.16
  only:
    - main
  cache:
    -
      key: PROD-PROMETHEUS-$CI_PIPELINE_ID
      paths:
        - $CI_PROJECT_DIR/monitor/kube-prometheus/default/manifests/
  before_script:
    - apk add --no-cache curl python3 py3-pip
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### ### kube-prometheus customizations
    - cd ./monitor/kube-prometheus/custom
    ### Edit Prometheus config to make it persistent using Storage Class
    - cp -f ./prometheus/* ../default/manifests
    ### Edit Grafana config to add Loki as datasource and make Grafana persistent with PVC
    - cp -f ./grafana/* ../default/manifests/
    ### Deploy kube-prometheus
    - cd ../default
    - kubectl apply -f manifests/
    ### ### Prometheus, Grafana and Alert Manager are now accessible using port-forwarding
    ### kubectl -n monitoring port-forward svc/prometheus-operated 9090
    ### kubectl -n monitoring port-forward svc/grafana 3000
    ### kubectl -n monitoring port-forward svc/alertmanager-main 9093

app tracing with jaeger:
  stage: prod monitor
  needs: ["monitoring prerequisites"]
  when: delayed
  start_in: 5 minutes
  image: alpine:3.16
  only:
    - main
  before_script:
    - apk add --no-cache curl python3 py3-pip
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Install Jaeger Operator
    - cd ./monitor/jaeger
    - kubectl apply -f jaeger-operator-role-binding.yml
    - kubectl create namespace observability --dry-run=client -o yaml | kubectl apply -f -
    - kubectl apply -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.35.0/jaeger-operator.yaml -n observability
    ### Create Elasticsearch production cluster
    - kubectl apply -f elasticsearch-prod.yml
    ### Create secret for Jaeger, based on Elasticsearch credentials
    - PASSWORD=$(kubectl get secret jaeger-es-elastic-user -o go-template='{{.data.elastic | base64decode}}')
    - kubectl create secret generic jaeger-es-secret --from-literal=ES_PASSWORD=$PASSWORD --from-literal=ES_USERNAME=elastic --dry-run=client -o yaml | kubectl apply -f -
    ### Wait for installation to finish
    - sleep 30
    ### Deploy Jaeger for Production
    - kubectl apply -f jaeger-prod.yml
    ### ### Jaeger is now accessible using port-forwarding
    ### kubectl port-forward svc/simple-prod-query 16686

log management with loki:
  stage: prod monitor
  needs: ["monitor kube with kube-prometheus"]
  when: on_success
  image: alpine:3.16
  only:
    - main
  before_script:
    - apk add --no-cache curl python3 py3-pip
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
    ### Install helm
    - apk add --no-cache tar
    - wget https://get.helm.sh/helm-v3.9.1-linux-amd64.tar.gz
    - tar -zxvf helm-v3.9.1-linux-amd64.tar.gz
    - chmod +x linux-amd64/helm
    - mv linux-amd64/helm /usr/local/bin/helm
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Install Loki
    - helm repo add grafana https://grafana.github.io/helm-charts
    - helm repo update
    - helm upgrade --install loki grafana/loki-stack --namespace monitoring --set loki.persistence.enabled=true,loki.persistence.storageClassName=ssd,loki.persistence.size=10Gi
    ### ### Loki is now linked to Grafana

integrate kubecost:
  stage: prod finops
  needs: ["log management with loki"]
  when: on_success
  image: alpine:3.16
  only:
    - main
  before_script:
    - apk add --no-cache curl python3 py3-pip
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
    ### Install helm
    - apk add --no-cache tar
    - wget https://get.helm.sh/helm-v3.9.1-linux-amd64.tar.gz
    - tar -zxvf helm-v3.9.1-linux-amd64.tar.gz
    - chmod +x linux-amd64/helm
    - mv linux-amd64/helm /usr/local/bin/helm
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Install Kubecost
    - |
      helm upgrade -i kubecost \
      oci://public.ecr.aws/kubecost/cost-analyzer --version 1.96.0 \
      --namespace kubecost --create-namespace \
      -f https://raw.githubusercontent.com/kubecost/cost-analyzer-helm-chart/develop/cost-analyzer/values-eks-cost-monitoring.yaml
    ### ### Kubecost is now accessible using port-forwarding
    ### kubectl port-forward --namespace kubecost deployment/kubecost-cost-analyzer 9090

### ### WARNING
### The next jobs are part of the 'prod destroy' stage.
### In a real use case, it is strongly discouraged to create stages/jobs destroying resources in production.
### Exceptionally for this practical project made for learning, this stage brings a comfort of use, especially for debugging.
### ###

destroy kubecost:
  stage: prod destroy
  needs: ["integrate kubecost"]
  when: manual
  image: alpine:3.16
  only:
    - main
  before_script:
    - apk add --no-cache curl python3 py3-pip
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
    ### Install helm
    - apk add --no-cache tar
    - wget https://get.helm.sh/helm-v3.9.1-linux-amd64.tar.gz
    - tar -zxvf helm-v3.9.1-linux-amd64.tar.gz
    - chmod +x linux-amd64/helm
    - mv linux-amd64/helm /usr/local/bin/helm
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Destroy Kubecost
    - helm uninstall kubecost --namespace kubecost

destroy goldilocks:
  stage: prod destroy
  needs: ["monitor vertical pod autoscaler using goldilocks"]
  when: manual
  image:
    name: golang:1.18-alpine3.16
    entrypoint: [""]
  only:
    - main
  before_script:
    - apk add --no-cache curl python3 py3-pip git bash
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Destroy Goldilocks
    - cd ./monitor/vpa
    - git clone https://github.com/FairwindsOps/goldilocks.git
    - cd goldilocks
    - kubectl -n goldilocks delete -f hack/manifests/dashboard
    - kubectl -n goldilocks delete -f hack/manifests/controller
    - kubectl delete namespace goldilocks

destroy kube-prometheus:
  stage: prod destroy
  needs: ["monitor kube with kube-prometheus"]
  when: manual
  image: alpine:3.16
  only:
    - main
  cache:
    -
      key: PROD-PROMETHEUS-$CI_PIPELINE_ID
      paths:
        - $CI_PROJECT_DIR/monitor/kube-prometheus/default/manifests/
  before_script:
    - apk add --no-cache curl python3 py3-pip
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Destroy kube-prometheus
    - cd ./monitor/kube-prometheus/default
    - kubectl delete --ignore-not-found=true -f manifests/

destroy jaeger:
  stage: prod destroy
  needs: ["app tracing with jaeger"]
  when: manual
  image: alpine:3.16
  only:
    - main
  before_script:
    - apk add --no-cache curl python3 py3-pip
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Destroy Jaeger
    - cd ./monitor/jaeger
    - kubectl delete -f jaeger-prod.yml
    - kubectl delete secret jaeger-es-secret
    - kubectl delete -f elasticsearch-prod.yml
    - kubectl delete -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.35.0/jaeger-operator.yaml -n observability
    - kubectl delete namespace observability
    - kubectl delete -f jaeger-operator-role-binding.yml

destroy loki:
  stage: prod destroy
  needs: ["log management with loki"]
  when: manual
  image: alpine:3.16
  only:
    - main
  before_script:
    - apk add --no-cache curl python3 py3-pip
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
    ### Install helm
    - apk add --no-cache tar
    - wget https://get.helm.sh/helm-v3.9.1-linux-amd64.tar.gz
    - tar -zxvf helm-v3.9.1-linux-amd64.tar.gz
    - chmod +x linux-amd64/helm
    - mv linux-amd64/helm /usr/local/bin/helm
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Destroy Loki
    - helm uninstall loki --namespace monitoring

destroy prod deploy:
  stage: prod destroy
  needs: ["deploy prod on eks"]
  when: manual
  image:
    name: hashicorp/terraform:1.2.2
    entrypoint: [""]
  only:
    - main
  environment:
    name: production
    action: stop
  before_script:
    - apk add --no-cache curl python3 py3-pip openssl
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
    ### Install Docker
    - apk add --no-cache docker openrc
    ### Install kubectl
    - curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
    - chmod +x ./kubectl
    - mv ./kubectl /usr/local/bin/kubectl
  script:
    ### Install EKS kubeconfig file locally
    - aws eks update-kubeconfig --name product-hunting-eks-cluster
    ### Destroy prod deploy
    - cd ./deploy/eks
    - |
      cat <<EOF | tee ./main.tfvars
      ecr_registry = "$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com"
      ci_commit_short_sha = "$CI_COMMIT_SHORT_SHA"
      EOF
    - terraform init
    - terraform destroy -var-file=main.tfvars -auto-approve

destroy prod infra:
  stage: prod destroy
  needs: ["destroy prod deploy"]
  when: manual
  image:
    name: hashicorp/terraform:1.2.2
    entrypoint: [""]
  only:
    - main
  before_script:
    - apk add --no-cache python3 py3-pip
    - pip3 install awscli
    - aws configure set region $AWS_REGION
  script:
    - cd ./infrastructure/prod
    - terraform init
    - terraform destroy -auto-approve

destroy unused volumes:
### Used to destroy unused volumes generated by PVC & SC once prod infra is destroyed 
  stage: prod destroy
  needs: ["destroy prod infra"]
  when: on_success
  image: alpine:3.16
  only:
    - main
  before_script:
    - apk add --no-cache curl python3 py3-pip
    ### Install awscli
    - pip3 install awscli
    - aws configure set region $AWS_REGION
  script:
    ### Destroy unused volumes
    - |
      LIST_UNUSED_VOL=$(aws ec2 describe-volumes \
        --filters Name=tag:kubernetes.io/created-for/pv/name,Values=pvc* \
        --query "Volumes[*].{ID:VolumeId}" --output text)
      for VOL in $LIST_UNUSED_VOL; do
        aws ec2 delete-volume --volume-id $VOL
      done