skip: add sequential parallel resources

.github/workflows/auto-test.yml

@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - "feature/policy_testing_v2"
+      - "feature/deploy_and_scan_sequential_parallel_resources"
   # Run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:
@@ -27,7 +28,7 @@ env:
   default_resource_priority_list: "['account', 'glue', 'sns']"
   #'[ "account", "acm", "airflow", "ami", "apigwv2", "app-elb", "app-flow", "asg", "backup", "cfn", "cloudtrail", "codebuild", "codedeploy", "codepipeline", "dax", "directory", "distribution", "dlm", "dms", "dynamodb", "ebs", "ec2", "ecr", "ecs", "efs", "eip", "eks", "elasticache", "elasticbeanstalk", "elasticsearch", "elb", "emr", "eni", "event", "firehose", "fsx", "glacier", "glue", "graphql", "hostedzone", "iam", "internet", "kafka", "key", "kinesis", "kms", "lambda", "launch", "lightsail", "log", "message", "nat", "network", "peering", "r53domain", "rds", "redshift", "rest", "route", "rrset", "s3", "sagemaker", "secrets", "security-group", "sns", "sqs", "step", "subnet", "transit", "vpc", "vpn", "waf"]'
   RED: '\033[0;31m'
-  ACTIONS_REPO_BRANCH: "main" 
+  ACTIONS_REPO_BRANCH: "feature/deploy_and_scan_sequential_parallel_resources" 
 
 permissions:
   contents: "read"
@@ -112,10 +113,12 @@ jobs:
     runs-on: ubuntu-22.04
     needs: deploy_common_resources
     outputs:
+      action_repo_name: ${{ steps.prepare-resource-matrix.outputs.action_repo_name }}
       parallel_resources_list: ${{ steps.prepare-resource-matrix.outputs.parallel_resources_to_scan }}
       not_parallel_resources_list: ${{ steps.prepare-resource-matrix.outputs.not_parallel_resources_to_scan }}
       sequential_resources_list: ${{ steps.prepare-resource-matrix.outputs.sequential_resources_to_scan }}
       sequential_resources_length: ${{ steps.prepare-resource-matrix.outputs.sequential_resources_length }}
+      sequential_parallel_resources_list: ${{ steps.prepare-resource-matrix.outputs.sequential_parallel_resources_to_scan }}
     steps:
       - name: Git clone the repository
         uses: actions/checkout@v4
@@ -129,123 +132,143 @@ jobs:
       - name: Prepare resource matrix
         id: prepare-resource-matrix
         uses: ./ecc-actions/auto-test-actions/prepare-resource-matrix
-
-  deploy_and_scan_parallel_resources:
-    name: Scan P
-    runs-on: ubuntu-22.04
-    needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] # pack_iam_policies_per_resource,
-    if: ${{ needs.prepare_resource_matrix.outputs.parallel_resources_list != '[]' }}
-    strategy:
-      max-parallel: 10
-      fail-fast: false
-      matrix:
-        compliance: ['green', 'red']
-        resource: ${{fromJson(needs.prepare_resource_matrix.outputs.parallel_resources_list)}}
-    env:
-      COMPLINCE: ${{ matrix.compliance }}
-      RESOURCE: ${{ matrix.resource }}
-    steps:
-      - name: Git clone the repository
-        uses: actions/checkout@v4
-
-      - name: Checkout ecc-actions
-        run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
         env:
-          PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
           ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }}
+
+  # deploy_and_scan_parallel_resources:
+  #   name: Scan P
+  #   runs-on: ubuntu-22.04
+  #   needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] # pack_iam_policies_per_resource,
+  #   if: ${{ needs.prepare_resource_matrix.outputs.parallel_resources_list != '[]' }}
+  #   strategy:
+  #     max-parallel: 10
+  #     fail-fast: false
+  #     matrix:
+  #       compliance: ['green', 'red']
+  #       resource: ${{fromJson(needs.prepare_resource_matrix.outputs.parallel_resources_list)}}
+  #   env:
+  #     COMPLINCE: ${{ matrix.compliance }}
+  #     RESOURCE: ${{ matrix.resource }}
+  #   steps:
+  #     - name: Git clone the repository
+  #       uses: actions/checkout@v4
+
+  #     - name: Checkout ecc-actions
+  #       run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
+  #       env:
+  #         PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
+  #         ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }}
 
-      - name: Deploy and scan parallel resources
-        uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
-        with:
-          CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
-          SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
-          WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
-          COMPLIANCE: ${{ matrix.compliance }}
-          PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
-          READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
-          GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}
+  #     - name: Deploy and scan parallel resources
+  #       uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
+  #       with:
+  #         CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
+  #         SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
+  #         WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
+  #         COMPLIANCE: ${{ matrix.compliance }}
+  #         PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
+  #         READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
+  #         GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}
 
-  deploy_and_scan_not_parallel_resources:
-    name: Scan N/P
-    runs-on: ubuntu-22.04
-    needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] # pack_iam_policies_per_resource,
-    if: ${{ needs.prepare_resource_matrix.outputs.not_parallel_resources_list != '[]' }}
-    strategy:
-      max-parallel: 1
-      fail-fast: false
-      matrix:
-        compliance: ['green', 'red']
-        resource: ${{fromJson(needs.prepare_resource_matrix.outputs.not_parallel_resources_list)}}
-    env:
-      COMPLINCE: ${{ matrix.compliance }}
-      RESOURCE: ${{ matrix.resource }}
-    steps:
-    - name: Git clone the repository
-      uses: actions/checkout@v4
+  # deploy_and_scan_not_parallel_resources:
+  #   name: Scan N/P
+  #   runs-on: ubuntu-22.04
+  #   needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] # pack_iam_policies_per_resource,
+  #   if: ${{ needs.prepare_resource_matrix.outputs.not_parallel_resources_list != '[]' }}
+  #   strategy:
+  #     max-parallel: 1
+  #     fail-fast: false
+  #     matrix:
+  #       compliance: ['green', 'red']
+  #       resource: ${{fromJson(needs.prepare_resource_matrix.outputs.not_parallel_resources_list)}}
+  #   env:
+  #     COMPLINCE: ${{ matrix.compliance }}
+  #     RESOURCE: ${{ matrix.resource }}
+  #   steps:
+  #   - name: Git clone the repository
+  #     uses: actions/checkout@v4
 
-    - name: Checkout ecc-actions
-      run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
-      env:
-        PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
-        ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }}
-    - name: Deploy and scan non-parallel resources
-      uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
-      with:
-        CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
-        SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
-        WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
-        COMPLIANCE: ${{ matrix.compliance }}
-        PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
-        READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
-        GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}
+  #   - name: Checkout ecc-actions
+  #     run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
+  #     env:
+  #       PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
+  #       ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }}
+  #   - name: Deploy and scan non-parallel resources
+  #     uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
+  #     with:
+  #       CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
+  #       SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
+  #       WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
+  #       COMPLIANCE: ${{ matrix.compliance }}
+  #       PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
+  #       READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
+  #       GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}
 
-  deploy_and_scan_sequential_resources:
-    name: Scan S
-    runs-on: ubuntu-22.04
+  # deploy_and_scan_sequential_resources:
+  #   name: Scan S
+  #   runs-on: ubuntu-22.04
+  #   needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix]
+  #   if: ${{ needs.prepare_resource_matrix.outputs.sequential_resources_list != '[]' }}
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       resource: ${{fromJson(needs.prepare_resource_matrix.outputs.sequential_resources_list)}}
+  #   env:
+  #     RESOURCE: ${{ matrix.resource }}
+  #   steps:
+  #   - name: Git clone the repository
+  #     uses: actions/checkout@v4
+
+  #   - name: Checkout ecc-actions
+  #     run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
+  #     env:
+  #       PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
+  #       ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }}
+
+  #   - name: Deploy and scan non-parallel resource (green)
+  #     uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
+  #     env:
+  #       COMPLINCE: "green"
+  #     with:
+  #       CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
+  #       SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
+  #       WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
+  #       COMPLIANCE: ${{ matrix.compliance }}
+  #       PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
+  #       READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
+  #       GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}
+
+  #   - name: Deploy and scan non-parallel resource (red)
+  #     uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
+  #     env:
+  #       COMPLINCE: "red"
+  #     if: always()
+  #     with:
+  #       CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
+  #       SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
+  #       WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
+  #       COMPLIANCE: ${{ matrix.compliance }}
+  #       PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
+  #       READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
+  #       GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}
+
+  deploy_and_scan_sequential_parallel_resources:
+    name: Scan S/P
     needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix]
-    if: ${{ needs.prepare_resource_matrix.outputs.sequential_resources_list != '[]' }}
     strategy:
+      max-parallel: 1
       fail-fast: false
       matrix:
-        resource: ${{fromJson(needs.prepare_resource_matrix.outputs.sequential_resources_list)}}
-    env:
+        resource: ${{fromJson(needs.prepare_resource_matrix.outputs.not_parallel_resources_list)}}
+        # resource: ${{fromJson(needs.prepare_resource_matrix.outputs.sequential_parallel_resources_list)}}
+    uses: ./.github/workflows/sequential_parallel_scan.yml
+    secrets: inherit
+    concurrency:
+      group: ${{ github.workflow }}
+    with:
       RESOURCE: ${{ matrix.resource }}
-    steps:
-    - name: Git clone the repository
-      uses: actions/checkout@v4
-
-    - name: Checkout ecc-actions
-      run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
-      env:
-        PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
-        ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }}
-
-    - name: Deploy and scan non-parallel resource (green)
-      uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
-      env:
-        COMPLINCE: "green"
-      with:
-        CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
-        SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
-        WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
-        COMPLIANCE: ${{ matrix.compliance }}
-        PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
-        READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
-        GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}
-
-    - name: Deploy and scan non-parallel resource (red)
-      uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
-      env:
-        COMPLINCE: "red"
-      if: always()
-      with:
-        CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
-        SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
-        WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
-        COMPLIANCE: ${{ matrix.compliance }}
-        PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
-        READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
-        GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}
+      ACTIONS_REPO_BRANCH: ${{ needs.prepare_resource_matrix.outputs.action_repo_name }}
+      READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
 
   delete_readonly_role_for_scans:
     name: Delete readonly role for scans

.github/workflows/sequential_parallel_scan.yml

@@ -0,0 +1,45 @@
+name: Deploy and scan sequential parallel resources
+
+on:
+  workflow_call:
+    inputs:
+      RESOURCE:
+        type: string
+      ACTIONS_REPO_BRANCH:
+        type: string
+      READONLY_ROLE_NAME:
+        type: string
+
+jobs:
+  deploy_and_scan_sequential_parallel_resources:
+    name: Scan S/P
+    runs-on: ubuntu-22.04
+    strategy:
+      max-parallel: 2
+      fail-fast: false
+      matrix:
+        compliance: ['green', 'red']
+    env: 
+      COMPLINCE: ${{ matrix.compliance }}
+    steps:
+      - name: Git clone the repository
+        uses: actions/checkout@v4
+
+      - name: Checkout ecc-actions
+        run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
+        env:
+          PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
+          ACTIONS_REPO_BRANCH: ${{ inputs.ACTIONS_REPO_BRANCH }}
+
+      - name: Deploy and scan sequential parallel resources
+        uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
+        with:
+          CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
+          SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
+          WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
+          COMPLIANCE: ${{ matrix.compliance }}
+          PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
+          READONLY_ROLE_NAME: ${{ inputs.READONLY_ROLE_NAME }}
+          GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}
+
+