semantics.yml

#
# Copyright (c) 2021-2022 T-Systems International GmbH (Catena-X Consortium)
#
# See the AUTHORS file(s) distributed with this work for additional
# information regarding authorship.
#
# See the LICENSE file(s) distributed with this work for
# additional information regarding license terms.
#

#################################################################################################################
# Github Workflow for continuous semantics deployment, could depend on infrastructure workflows
#################################################################################################################
# The following secrets are expected to be set
# AZURE_CREDENTIALS           - JSON object containing all info to login to azure via a Service Principal (SP)
# AZURE_SUBSCRIPTION_ID       - part of AZURE_CREDENTIALS referring the ID of the deployment target
# AZURE_AD_CLIENT_ID          - part of AZURE_CREDENTIALS referring the ID of the SP
# AZURE_AD_CLIENT_SECRET      - part of AZURE_CREDENTIALS having the login secret for the SP
# AZURE_AD_TENANT_ID          - part of AZURE_CREDENTIALS referring the hosting AD of the SP
# CATENAX_ADMIN_USER          - name of the admin user to be used to secure deployed resources
# CATENAX_ADMIN_PASSWORD      - secret of the admin user to be used to secure deployed resources
# CATENAX_USER                - name of the operation user to be used to secure deployed resources
# CATENAX_PASSWORD            - secret of the operation user to be used to secure deployed resources
# CATENAX_ADMIN_GROUP_ID      - referring to the ID of the admin group that will be assigned to deployed resources
#################################################################################################################
# The following preconditions need to be made
# Upload ${workspace}ssh.pub to the TFSTATE storage account
##################################################################################################################
# If you intend to use this workflow in new branches/workspaces, see the comments marked 
# ADAPT HERE TO ADD NEW WORKSPACES

# name of workflow (as well as subordinate workflows and jobs) start with a "run-level" such that we can
# depend on them in order to implement workflow dependencies
name: 4 Services Semantics

####################################################
# Should trigger upon pushs to the portal related 
# branches and folders
####################################################
# ADAPT HERE TO ADD NEW WORKSPACES

on: 
  push:
    paths:
      # This is where the terraform related scripts and manifests reside
      - 'semantics/**'
      # any deployment manifest
      - 'infrastructure/manifests/semantics.yaml'
      # this workflow file
      - '.github/workflows/semantics.yml'
      # ADAPT HERE TO ADD NEW WORKSPACES
    branches:
      # Integration environment
      - main
      # ART1 Semantics Layer and Digitial Twin Dev Space
      - feature/CATX-A1-SLDT
      # ADAPT HERE TO ADD NEW WORKSPACES

###############################################################
# Consists of two jobs, the first determines the environment
# and the second one (doing the real work) only triggers
# if such an environment can be detected (else the workflow is
# green without doing something)
###############################################################
# ADAPT HERE TO ADD NEW WORKSPACES

jobs:

  ########################################
  # First job to determine the environment
  ########################################
  # ADAPT HERE TO ADD NEW WORKSPACES

  environment:
    # name of the job starts with a "run-level" subordinate to the workflow such that we can
    # depend on them in order to implement workflow dependencies
    name: 40 Services Semantics Determine Target Environment
    runs-on: ubuntu-latest
    outputs:
      workspace: ${{ steps.setvars.outputs.workspace }}

    steps:
      - name: Set variables
        id: setvars
        run: |
          if [[ "${{github.repository}}" == eclipse/tractusx ]]; then
             if [[ "${{github.ref}}" == refs/heads/main ]]; then
                echo "Determined PRODUCTION"
                echo "::set-output name=workspace::prod"
             else
                echo "Unsupported Environment on ECLIPSE. Leaving Workspace empty."
             fi
          else 
            if [[ "${{github.repository}}" == catenax/tractusx ]]; then
              if [[ "${{github.ref}}" == refs/heads/main ]]; then
                echo "Determined INTEGRATION"
                echo "::set-output name=workspace::int"
              elif [[ "${{github.ref}}" = refs/heads/feature/CATX-A1-SLDT ]]; then
                  echo "Determined SEMANTICS"
                  echo "::set-output name=workspace::dev042"
              else 
                  echo "Unsupported Branch on CATENAX. Leaving Workspace empty."
              fi
            else
                echo "Unsupported Environment/Repository. Leaving Workspace empty."
            fi
          fi

  ##########################################
  # Second job does the real terraform work
  ##########################################
  # FROM HERE ON: NO ADAPTION NEEDED ANYMORE

  containerize:
    # name of the job starts with a "run-level" subordinate to the workflow such that we can
    # depend on them in order to implement workflow dependencies
    name: 49 Services Semantics Build & Deploy
    runs-on: ubuntu-latest
    # rely on the first job
    needs: environment
    # rely on successful detection of the workspace, ignore if empty
    if: ${{needs.environment.outputs.workspace}}
    # Set important environment vars
    env:
      WORKSPACE: ${{needs.environment.outputs.workspace}}
      CONTAINER_REGISTRY_SHORT: catenaxacr
      CONTAINER_REGISTRY: catenaxacr.azurecr.io
      K8_RESOURCE_GROUP: catenax-${{needs.environment.outputs.workspace}}-rg
      CATENA_PORTAL_URL: catenax${{needs.environment.outputs.workspace}}aksportal.germanywestcentral.cloudapp.azure.com
      CATENA_SERVICE_URL: catenax${{needs.environment.outputs.workspace}}akssrv.germanywestcentral.cloudapp.azure.com
      # use the commit hash in production?
      VERSION: latest
      # User never in production?
      IMAGE_PULL_POLICY: Always
      IMAGE_SEMANTIC_HUB: "catenaxacr.azurecr.io/semantics/hub${{needs.environment.outputs.workspace}}:latest"
      IMAGE_REGISTRY: "catenaxacr.azurecr.io/semantics/registry${{needs.environment.outputs.workspace}}:latest"
      IMAGE_ADAPTER: "catenaxacr.azurecr.io/semantics/adapter${{needs.environment.outputs.workspace}}:latest"
      GRAPHDB_SHARE_NAME: catenax${{needs.environment.outputs.workspace}}graphdb
      IDP_ISSUER_URI: "https://catenax${{needs.environment.outputs.workspace}}akssrv.germanywestcentral.cloudapp.azure.com/iamcentralidp/auth/realms/CX-Central"
    steps:

    # Wait for infrastructure deployments to terminate
    - name: Wait for Infrastructure
      uses: lewagon/wait-on-check-action@v1.0.0
      with:
        ref: ${{ github.ref }}
        check-regexp: 0.*
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        wait-interval: 30
        allowed-conclusions: success,skipped,cancelled


    # Checks-out this repo/branch under $GITHUB_WORKSPACE/tractusx, so your job can access it
    # get the latest sources
    - name: Checkout
      uses: actions/checkout@v2

    # This is needed to interact with the subscription
    - name: Login to Azure
      uses: azure/login@v1
      with:
        creds: ${{ secrets.AZURE_CREDENTIALS }}

    # Login to the ACR
    - name: Login to Azure Container Registry
      run: |
        az acr login --name ${{ env.CONTAINER_REGISTRY_SHORT }}

    # Login to K8
    - name: Kubernetes Login
      uses: azure/aks-set-context@v1
      with:
        creds: '${{ secrets.AZURE_CREDENTIALS }}' # Azure credentials
        resource-group: 'catenax-${{needs.environment.outputs.workspace}}-rg'
        cluster-name: 'catenax-${{needs.environment.outputs.workspace}}-aks-services'
      id: login
         
    # Configure Secrets
    - name: Update Kubernetes Secrets
      run: |
        kubectl create secret generic semantics-secret -n semantics --from-literal=database_url=jdbc:postgresql://catenax${{needs.environment.outputs.workspace}}database.postgres.database.azure.com:5432/semantics?sslmode=require --from-literal=database_user=${{ secrets.CATENAX_ADMIN_USER }}@catenax${{needs.environment.outputs.workspace}}database --from-literal=database_password=${{ secrets.CATENAX_ADMIN_PASSWORD }} --from-literal=http_basic_auth_user=${{ secrets.CATENAX_USER }} --from-literal=http_basic_auth_password=${{ secrets.CATENAX_PASSWORD }} --from-literal=connector_user=${{ secrets.CATENAX_ADMIN_USER }} --from-literal=connector_password=${{ secrets.CATENAX_ADMIN_PASSWORD }} --from-literal=idp_issuer_uri=${{env.IDP_ISSUER_URI}} --dry-run=client -o yaml | kubectl apply -f -
        kubectl create secret generic graphdb-share-secret -n semantics --from-literal=azurestorageaccountname=catenax${{needs.environment.outputs.workspace}}storage --from-literal=azurestorageaccountkey=$(az storage account keys list --account-name catenax${{needs.environment.outputs.workspace}}storage --query '[0].value' -o tsv) --dry-run=client -o yaml | kubectl apply -f -

    - name: Build & Containerize the Layer (GraphDb)
      working-directory: ./
      run: |
        queryResult=$(docker manifest inspect $CONTAINER_REGISTRY/semantics/jena-fuseki:4.1.0 > /dev/null ; echo $?)
        if [[ $queryResult == '0' ]]; then
          echo "Image $CONTAINER_REGISTRY/semantics/jena-fuseki:4.1.0 does already exists. Nothing to do."
        else
          curl https://repo1.maven.org/maven2/org/apache/jena/jena-fuseki-docker/4.1.0/jena-fuseki-docker-4.1.0.zip --output jena-fuseki-docker-4.1.0.zip
          unzip -o jena-fuseki-docker-4.1.0.zip
          docker build jena-fuseki-docker-4.1.0/ --build-arg JENA_VERSION=4.1.0 -t $CONTAINER_REGISTRY/semantics/jena-fuseki:4.1.0
          docker push $CONTAINER_REGISTRY/semantics/jena-fuseki:4.1.0
        fi
         
    - name: Build the Layer (Maven)
      working-directory: ./semantics
      env:
        GITHUB_PACKAGE_ACCESS_TENANT: ${{ secrets.SEMANTICS_PACKAGE_ACCESS_TENANT }}
        GITHUB_PACKAGE_ACCESS_TOKEN: ${{ secrets.SEMANTICS_PACKAGE_ACCESS_TOKEN }}
      run: |
        mvn -s ../.github/workflows/settings.xml clean install -Dmaven.javadoc.skip=true

    # TODO: uses npm as the server for debugging purposes
    - name: Containerize the Layer (ServicesApp)
      working-directory: ./semantics
      run: |
        docker build -t ${{ env.IMAGE_SEMANTIC_HUB }} ./semantic-hub
        docker build -t ${{ env.IMAGE_REGISTRY }} ./registry
        
    - name: Publish the Layer (ServicesApp)
      working-directory: ./semantics
      run: |
        docker push ${{ env.IMAGE_SEMANTIC_HUB }}
        docker push ${{ env.IMAGE_REGISTRY }}

    # TODO: uses npm as the server for debugging purposes
    - name: Containerize the Layer (AdapterApp)
      working-directory: ./semantics
      run: |
        docker build -f ./adapter/Dockerfile -t ${{ env.IMAGE_ADAPTER }} .
        
    - name: Publish the Layer (AdapterApp)
      working-directory: ./semantics
      run: |
        docker push ${{ env.IMAGE_ADAPTER }}

    - name: Deploy the Layer
      working-directory: ./infrastructure/manifests
      run: |
        cat semantics.yaml | envsubst | kubectl apply -n semantics -f -

    # Get Pending Solver, we have to wait a bit as it may take some time for the solver to come up
    # We need two Solver as we have two separate docker images
    - name: Kubernetes Try To Find Pending Solver
      id: try_solver1
      working-directory: ./infrastructure/terraform
      run: |
        sleep 15s
        echo "::set-output name=SOLVER::$(kubectl describe ingress -n semantics cm-acme | sed -n 's/Name:[\w]*\([\S]*\)/\1/p')"

    - name: Modify a pending <1.18 cert-manager solver
      working-directory: ./infrastructure/manifests
      if: "${{steps.try_solver1.outputs.SOLVER}}"
      run: |
        echo "Found a pending solver1 ${{steps.try_solver1.outputs.SOLVER}} which we need to patch"
        kubectl get ingress ${{steps.try_solver1.outputs.SOLVER}} -n semantics -o yaml | sed '/^.*kubernetes\.io\/ingress\.class:.*service.*$/d' | sed "/^spec:$/a\  ingressClassName: service" | kubectl apply -f -

    # Get Pending Solver, we have to wait a bit as it may take some time for the solver to come up
    # We need two Solver as we have two separate docker images
    - name: Kubernetes Try To Find Pending Solver
      id: try_solver2
      working-directory: ./infrastructure/terraform
      run: |
        sleep 15s
        echo "::set-output name=SOLVER::$(kubectl describe ingress -n semantics cm-acme | sed -n 's/Name:[\w]*\([\S]*\)/\1/p')"

    - name: Modify a pending <1.18 cert-manager solver
      working-directory: ./infrastructure/manifests
      if: "${{steps.try_solver2.outputs.SOLVER}}"
      run: |
        echo "Found a pending solver2 ${{steps.try_solver2.outputs.SOLVER}} which we need to patch"
        kubectl get ingress ${{steps.try_solver2.outputs.SOLVER}} -n semantics -o yaml | sed '/^.*kubernetes\.io\/ingress\.class:.*service.*$/d' | sed "/^spec:$/a\  ingressClassName: service" | kubectl apply -f -



    - name: Rollout the Layer
      working-directory: ./infrastructure/manifests
      run: |
        kubectl rollout restart deployment registry -n semantics
        kubectl rollout restart deployment semantic-hub -n semantics
        kubectl rollout restart deployment adapter -n semantics