Add CD workflow.

.github/workflows/_comps-workflow.yml

@@ -0,0 +1,137 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Comps jobs
+permissions: read-all
+on:
+  workflow_call:
+    inputs:
+      # node:
+      #   required: true
+      #   type: string
+      service:
+        required: true
+        type: string
+      name:
+        required: true
+        type: string
+      tag:
+        default: "comps"
+        required: false
+        type: string
+      # build:
+      #   default: true
+      #   required: false
+      #   type: boolean
+      # scan:
+      #   default: true
+      #   required: false
+      #   type: boolean
+      # test_compose:
+      #   default: false
+      #   required: false
+      #   type: boolean
+      # test_k8s:
+      #   default: false
+      #   required: false
+      #   type: boolean
+      # GenAIComps_branch:
+      #   default: "main"
+      #   required: false
+      #   type: string
+jobs:
+  ####################################################################################################
+  # Image Build
+  ####################################################################################################
+  build-images:
+    runs-on: "docker-build-gaudi"
+    continue-on-error: true
+    steps:
+      - name: Clean Up Working Directory
+        run: sudo rm -rf ${{github.workspace}}/*
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+
+      - name: Clone required Repo
+        run: |
+          cd ${{ github.workspace }}/.github/workflows/docker/compose
+          docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
+          if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then
+              git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4
+          fi
+          if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then
+              git clone https://github.com/vllm-project/vllm.git vllm-openvino
+          fi
+
+      - name: Build Image
+        if: ${{ fromJSON(inputs.build) }}
+        uses: opea-project/validation/actions/image-build@main
+        with:
+          work_dir: ${{ github.workspace }}/.github/workflows/docker/compose/
+          docker_compose_yml: ${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
+          registry: ${OPEA_IMAGE_REPO}opea
+          tag: ${{ inputs.tag }}
+
+  ####################################################################################################
+  # Trivy Scan
+  ####################################################################################################
+  # get-image-list:
+  #   needs: [build-images]
+  #   if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi' }}
+  #   runs-on: ubuntu-latest
+  #   outputs:
+  #     matrix: ${{ steps.scan-matrix.outputs.matrix }}
+  #   steps:
+  #     - name: Checkout out Repo
+  #       uses: actions/checkout@v4
+
+  #     - name: Set Matrix
+  #       id: scan-matrix
+  #       run: |
+  #         pip install yq
+  #         compose_path=${{ github.workspace }}/${{ inputs.service }}/docker/docker_build_compose.yaml
+  #         echo "matrix=$(cat ${compose_path} | yq -r '.[]' | jq 'keys' | jq -c '.')" >> $GITHUB_OUTPUT
+
+  # scan-images:
+  #   needs: [get-image-list, build-images]
+  #   if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi'}}
+  #   runs-on: "docker-build-${{ inputs.node }}"
+  #   strategy:
+  #     matrix:
+  #       image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}
+  #     fail-fast: false
+  #   steps:
+  #     - name: Pull Image
+  #       run: |
+  #         docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+  #         echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV
+
+  #     - name: Scan Container
+  #       uses: opea-project/validation/actions/trivy-scan@main
+  #       with:
+  #         image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
+  #         output: ${{ matrix.image }}-scan.txt
+
+  #     - name: Cleanup
+  #       if: always()
+  #       run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
+
+  #     - uses: actions/upload-artifact@v4.3.4
+  #       with:
+  #         name: ${{ matrix.image }}-scan
+  #         path: ${{ matrix.image }}-scan.txt
+  #         overwrite: true
+
+  ####################################################################################################
+  # Docker Compose Test
+  ####################################################################################################
+  test-service-compose:
+    needs: [build-images]
+    if: ${{ fromJSON(inputs.test_compose) }}
+    uses: ./.github/workflows/_run-docker-compose.yml
+    with:
+      tag: ${{ inputs.tag }}
+      service: ${{ inputs.service }}
+      name: ${{ inputs.name }}
+    secrets: inherit

.github/workflows/_run-docker-compose.yml

@@ -0,0 +1,117 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Image Build
+permissions: read-all
+on:
+  workflow_call:
+    inputs:
+      # registry:
+      #   description: Container Registry URL
+      #   required: false
+      #   default: ""
+      #   type: string
+      tag:
+        description: Container Tag
+        required: false
+        default: "comps"
+        type: string
+      service:
+        description: Service to test
+        required: true
+        type: string
+      name:
+        description: Name to run the test on
+        required: true
+        type: string
+jobs:
+  get-test-case:
+    runs-on: ubuntu-latest
+    outputs:
+      test_cases: ${{ steps.test-case-matrix.outputs.test_cases }}
+      CHECKOUT_REF: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
+    steps:
+      - name: Get checkout ref
+        id: get-checkout-ref
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
+            CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
+          else
+            CHECKOUT_REF=${{ github.ref }}
+          fi
+          echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
+          echo "checkout ref ${CHECKOUT_REF}"
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
+          fetch-depth: 0
+
+      - name: Get test matrix
+        shell: bash
+        id: test-case-matrix
+        run: |
+          set -x
+          service_l=$(echo ${{ inputs.service }} | tr '[:upper:]' '[:lower:]')
+          cd ${{ github.workspace }}/${{ inputs.service }}/tests
+          test_cases=$(find . -type f -name "test_${service_l}*on_${{ inputs.name }}.sh" -print | cut -d/ -f2 | jq -R '.' | jq -sc '.')
+          echo "test_cases=$test_cases" >> $GITHUB_OUTPUT
+
+  run-test:
+    needs: [get-test-case]
+    strategy:
+      matrix:
+        test_case: ${{ fromJSON(needs.get-test-case.outputs.test_cases) }}
+      fail-fast: false
+    runs-on: gaudi
+    continue-on-error: true
+    steps:
+      - name: Clean up Working Directory
+        run: |
+          sudo rm -rf ${{github.workspace}}/* || true
+          docker system prune -f
+          docker rmi $(docker images --filter reference="*/*/*:latest" -q) || true
+          docker rmi $(docker images --filter reference="*/*:ci" -q) || true
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.get-test-case.outputs.CHECKOUT_REF }}
+          fetch-depth: 0
+
+      - name: Run test
+        shell: bash
+        env:
+          HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
+          GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
+          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+          PINECONE_KEY: ${{ secrets.PINECONE_KEY }}
+          IMAGE_REPO: ${{ inputs.registry }}
+          IMAGE_TAG: ${{ inputs.tag }}
+          service: ${{ inputs.service }}
+          name: ${{ inputs.name }}
+          test_case: ${{ matrix.test_case }}
+        run: |
+          cd ${{ github.workspace }}/$service/tests
+          if [[ "$IMAGE_REPO" == "" ]]; then export IMAGE_REPO="${OPEA_IMAGE_REPO}opea"; fi
+          if [ -f ${test_case} ]; then timeout 30m bash ${test_case}; else echo "Test script {${test_case}} not found, skip test!"; fi
+
+      - name: Clean up container
+        shell: bash
+        if: cancelled() || failure()
+        run: |
+          cd ${{ github.workspace }}/${{ inputs.service }}/docker/${{ inputs.name }}
+          yaml_files=$(find . -type f -name "*compose*yaml")
+          for file in $yaml_files; do
+              docker compose -f ${file} stop && docker compose -f ${file} rm -f || true
+          done
+          docker system prune -f
+          docker rmi $(docker images --filter reference="*:5000/*/*" -q) || true
+
+      - name: Publish pipeline artifact
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.test_case }}
+          path: ${{ github.workspace }}/${{ inputs.service }}/tests/*.log

.github/workflows/manual-comps-workflow.yml

@@ -0,0 +1,77 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+name: Comps CD workflow on manual event
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, reopened, ready_for_review, synchronize]
+
+  workflow_dispatch:
+    inputs:
+      service:
+        default: "asr"
+        description: "List of services to test [agent,asr,chathistory,dataprep,embeddings,guardrails,knowledgegraphs,llms,lvms,prompt_registry,ragas,reranks,retrievers,tts,vectorstores,web_retrievers]"
+        required: true
+        type: string
+      name:
+        default: "comps"
+        description: "List of names to test [_faq-generation,_ls tefastrag,_haystack,_langchain,_langchain-mosec,_llama,_llava,_milvus,_mongo,_pgvector,_pii,_pinecone,_qdrant,_redis,_registry,_retrievers,_summarization,_tei,_text-generation,_tgi]"
+        required: true
+        type: string
+      tag:
+        default: "comps"
+        description: "Tag to apply to images"
+        required: true
+        type: string
+      # build:
+      #   default: true
+      #   description: 'Build test required images for Comps'
+      #   required: false
+      #   type: boolean
+      # scan:
+      #   default: true
+      #   description: 'Scan all images with Trivy'
+      #   required: false
+      #   type: boolean
+      # test_compose:
+      #   default: true
+      #   description: 'Test services with docker compose'
+      #   required: false
+      #   type: boolean
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  get-test-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      services: "dataprep" #${{ steps.get-matrix.outputs.services }}
+      names: "_pgvector" #${{ steps.get-matrix.outputs.names }}
+      steps:
+        - name: run workflow
+          id: get-matrix
+          run: |
+            echo "workflow is triggered"
+
+  run-services:
+    needs: [get-test-matrix]
+    strategy:
+      matrix:
+        service: ${{ fromJson(needs.get-test-matrix.outputs.services) }}
+        # node: ${{ fromJson(needs.get-test-matrix.outputs.nodes) }}
+        name: ${{ fromJson(needs.get-test-matrix.outputs.names }}
+      fail-fast: false
+    uses: ./.github/workflows/_comps-workflow.yml
+    with:
+      service: ${{ matrix.service }}
+      name: ${{ matrix.name }}
+      tag: "comps" #${{ inputs.tag }}
+      # build: ${{ fromJSON(inputs.build) }}
+      # scan: ${{ fromJSON(inputs.scan) }}
+      # test_compose: ${{ fromJSON(inputs.test_compose) }}
+
+      # GenAIComps_branch: ${{ inputs.GenAIComps_branch }}
+    secrets: inherit