skip: add sequential parallel resources #248
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Auto-testing | ||
| on: | ||
| push: | ||
| branches: | ||
| - "feature/policy_testing_v2" | ||
| - "feature/deploy_and_scan_sequential_parallel_resources" | ||
| # Run this workflow manually from the Actions tab | ||
| workflow_dispatch: | ||
| inputs: | ||
| resource_priority_list: | ||
| type: string | ||
| description: Priority list for resources (you can remove unnecessary resources during testing) | ||
| default: '[ ]' | ||
| required: true | ||
| # limits to only one workflow in time | ||
| concurrency: | ||
| group: ${{ github.workflow }} | ||
| env: | ||
| AUTO_TEST_DIR: "auto_policy_testing" | ||
| TF_VAR_project: ${{ secrets.TF_VAR_project }} | ||
| TF_VAR_region: ${{ secrets.AWS_REGION }} | ||
| TF_VAR_zone: ${{ secrets.TF_VAR_zone }} | ||
| TF_BACKEND_STORAGE_NAME: ${{ secrets.TF_BACKEND_STORAGE_NAME }} | ||
| TF_CLI_ARGS: "-no-color" | ||
| AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} | ||
| default_resource_priority_list: "['account', 'glue', 'sns']" | ||
| #'[ "account", "acm", "airflow", "ami", "apigwv2", "app-elb", "app-flow", "asg", "backup", "cfn", "cloudtrail", "codebuild", "codedeploy", "codepipeline", "dax", "directory", "distribution", "dlm", "dms", "dynamodb", "ebs", "ec2", "ecr", "ecs", "efs", "eip", "eks", "elasticache", "elasticbeanstalk", "elasticsearch", "elb", "emr", "eni", "event", "firehose", "fsx", "glacier", "glue", "graphql", "hostedzone", "iam", "internet", "kafka", "key", "kinesis", "kms", "lambda", "launch", "lightsail", "log", "message", "nat", "network", "peering", "r53domain", "rds", "redshift", "rest", "route", "rrset", "s3", "sagemaker", "secrets", "security-group", "sns", "sqs", "step", "subnet", "transit", "vpc", "vpn", "waf"]' | ||
| RED: '\033[0;31m' | ||
| ACTIONS_REPO_BRANCH: "feature/deploy_and_scan_sequential_parallel_resources" | ||
| permissions: | ||
| contents: "read" | ||
| id-token: "write" | ||
| jobs: | ||
| deploy_common_resources: | ||
| name: Deploy common | ||
| runs-on: ubuntu-22.04 | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| compliance: ["green", "red"] | ||
| env: | ||
| COMPLINCE: ${{ matrix.compliance }} | ||
| RESOURCE: common_resources | ||
| steps: | ||
| - name: Git clone the repository | ||
| uses: actions/checkout@v4 | ||
| - name: Checkout ecc-actions | ||
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | ||
| env: | ||
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | ||
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | ||
| - name: Deploy common | ||
| uses: ./ecc-actions/auto-test-actions/deploy-common-resources | ||
| with: | ||
| CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | ||
| SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | ||
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | ||
| COMPLIANCE: ${{ matrix.compliance }} | ||
| create_readonly_role_for_scans: | ||
| name: Create readonly role for scans | ||
| if: github.repository == 'epam/ecc-aws-rulepack' | ||
| runs-on: ubuntu-22.04 | ||
| needs: deploy_common_resources | ||
| outputs: | ||
| readonly_role_name: ${{ steps.create-readonly-role.outputs.readonly_role_name }} | ||
| steps: | ||
| - name: Git clone the repository | ||
| uses: actions/checkout@v4 | ||
| - name: Checkout ecc-actions | ||
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | ||
| env: | ||
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | ||
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | ||
| - name: Create readonly role for scans | ||
| id: create-readonly-role | ||
| uses: ./ecc-actions/auto-test-actions/readonly-role-for-scans | ||
| with: | ||
| CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | ||
| SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | ||
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | ||
| COMPLIANCE: ${{ matrix.compliance }} | ||
| PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }} | ||
| ROLE_ACTION: "create" | ||
| # pack_iam_policies_per_resource: | ||
| # runs-on: ubuntu-22.04 | ||
| # needs: deploy_common_resources | ||
| # steps: | ||
| # - name: Git clone the repository | ||
| # uses: actions/checkout@v4 | ||
| # - name: pack_iam_policies | ||
| # shell: bash | ||
| # working-directory: ./ecc-actions/auto-test-actions/scripts | ||
| # run: python -u pack_iam.py ${{ github.repository }} | ||
| # - name: Archive loggs to artifacts | ||
| # uses: actions/upload-artifact@v4 | ||
| # with: | ||
| # name: pack_iam | ||
| # path: | | ||
| # ${{ env.AUTO_TEST_DIR }}/iam | ||
| prepare_resource_matrix: | ||
| name: Prepare resource matrix | ||
| runs-on: ubuntu-22.04 | ||
| needs: deploy_common_resources | ||
| outputs: | ||
| action_repo_name: ${{ steps.prepare-resource-matrix.outputs.action_repo_name }} | ||
| parallel_resources_list: ${{ steps.prepare-resource-matrix.outputs.parallel_resources_to_scan }} | ||
| not_parallel_resources_list: ${{ steps.prepare-resource-matrix.outputs.not_parallel_resources_to_scan }} | ||
| sequential_resources_list: ${{ steps.prepare-resource-matrix.outputs.sequential_resources_to_scan }} | ||
| sequential_resources_length: ${{ steps.prepare-resource-matrix.outputs.sequential_resources_length }} | ||
| sequential_parallel_resources_list: ${{ steps.prepare-resource-matrix.outputs.sequential_parallel_resources_to_scan }} | ||
| steps: | ||
| - name: Git clone the repository | ||
| uses: actions/checkout@v4 | ||
| - name: Checkout ecc-actions | ||
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | ||
| env: | ||
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | ||
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | ||
| - name: Prepare resource matrix | ||
| id: prepare-resource-matrix | ||
| uses: ./ecc-actions/auto-test-actions/prepare-resource-matrix | ||
| env: | ||
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | ||
| # deploy_and_scan_parallel_resources: | ||
| # name: Scan P | ||
| # runs-on: ubuntu-22.04 | ||
| # needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] # pack_iam_policies_per_resource, | ||
| # if: ${{ needs.prepare_resource_matrix.outputs.parallel_resources_list != '[]' }} | ||
| # strategy: | ||
| # max-parallel: 10 | ||
| # fail-fast: false | ||
| # matrix: | ||
| # compliance: ['green', 'red'] | ||
| # resource: ${{fromJson(needs.prepare_resource_matrix.outputs.parallel_resources_list)}} | ||
| # env: | ||
| # COMPLINCE: ${{ matrix.compliance }} | ||
| # RESOURCE: ${{ matrix.resource }} | ||
| # steps: | ||
| # - name: Git clone the repository | ||
| # uses: actions/checkout@v4 | ||
| # - name: Checkout ecc-actions | ||
| # run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | ||
| # env: | ||
| # PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | ||
| # ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | ||
| # - name: Deploy and scan parallel resources | ||
| # uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources | ||
| # with: | ||
| # CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | ||
| # SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | ||
| # WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | ||
| # COMPLIANCE: ${{ matrix.compliance }} | ||
| # PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }} | ||
| # READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | ||
| # GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }} | ||
| # deploy_and_scan_not_parallel_resources: | ||
| # name: Scan N/P | ||
| # runs-on: ubuntu-22.04 | ||
| # needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] # pack_iam_policies_per_resource, | ||
| # if: ${{ needs.prepare_resource_matrix.outputs.not_parallel_resources_list != '[]' }} | ||
| # strategy: | ||
| # max-parallel: 1 | ||
| # fail-fast: false | ||
| # matrix: | ||
| # compliance: ['green', 'red'] | ||
| # resource: ${{fromJson(needs.prepare_resource_matrix.outputs.not_parallel_resources_list)}} | ||
| # env: | ||
| # COMPLINCE: ${{ matrix.compliance }} | ||
| # RESOURCE: ${{ matrix.resource }} | ||
| # steps: | ||
| # - name: Git clone the repository | ||
| # uses: actions/checkout@v4 | ||
| # - name: Checkout ecc-actions | ||
| # run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | ||
| # env: | ||
| # PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | ||
| # ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | ||
| # - name: Deploy and scan non-parallel resources | ||
| # uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources | ||
| # with: | ||
| # CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | ||
| # SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | ||
| # WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | ||
| # COMPLIANCE: ${{ matrix.compliance }} | ||
| # PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }} | ||
| # READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | ||
| # GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }} | ||
| # deploy_and_scan_sequential_resources: | ||
| # name: Scan S | ||
| # runs-on: ubuntu-22.04 | ||
| # needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] | ||
| # if: ${{ needs.prepare_resource_matrix.outputs.sequential_resources_list != '[]' }} | ||
| # strategy: | ||
| # fail-fast: false | ||
| # matrix: | ||
| # resource: ${{fromJson(needs.prepare_resource_matrix.outputs.sequential_resources_list)}} | ||
| # env: | ||
| # RESOURCE: ${{ matrix.resource }} | ||
| # steps: | ||
| # - name: Git clone the repository | ||
| # uses: actions/checkout@v4 | ||
| # - name: Checkout ecc-actions | ||
| # run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | ||
| # env: | ||
| # PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | ||
| # ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | ||
| # - name: Deploy and scan non-parallel resource (green) | ||
| # uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources | ||
| # env: | ||
| # COMPLINCE: "green" | ||
| # with: | ||
| # CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | ||
| # SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | ||
| # WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | ||
| # COMPLIANCE: ${{ matrix.compliance }} | ||
| # PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }} | ||
| # READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | ||
| # GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }} | ||
| # - name: Deploy and scan non-parallel resource (red) | ||
| # uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources | ||
| # env: | ||
| # COMPLINCE: "red" | ||
| # if: always() | ||
| # with: | ||
| # CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | ||
| # SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | ||
| # WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | ||
| # COMPLIANCE: ${{ matrix.compliance }} | ||
| # PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }} | ||
| # READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | ||
| # GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }} | ||
| deploy_and_scan_sequential_parallel_resources: | ||
| name: Scan S/P | ||
| needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] | ||
| strategy: | ||
| max-parallel: 1 | ||
| fail-fast: false | ||
| matrix: | ||
| resource: ${{fromJson(needs.prepare_resource_matrix.outputs.not_parallel_resources_list)}} | ||
| # resource: ${{fromJson(needs.prepare_resource_matrix.outputs.sequential_parallel_resources_list)}} | ||
| uses: ./.github/workflows/sequential_parallel_scan.yml | ||
| secrets: inherit | ||
| concurrency: | ||
| group: ${{ github.workflow }} | ||
| with: | ||
| RESOURCE: ${{ matrix.resource }} | ||
| ACTIONS_REPO_BRANCH: ${{ needs.prepare_resource_matrix.outputs.action_repo_name }} | ||
| READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | ||
| delete_readonly_role_for_scans: | ||
| name: Delete readonly role for scans | ||
| if: ${{ always() }} | ||
| runs-on: ubuntu-22.04 | ||
| needs: [ create_readonly_role_for_scans, deploy_and_scan_parallel_resources, deploy_and_scan_not_parallel_resources, deploy_and_scan_sequential_resources ] | ||
|
Check failure on line 277 in .github/workflows/auto-test.yml
|
||
| steps: | ||
| - name: Git clone the repository | ||
| uses: actions/checkout@v4 | ||
| - name: Checkout ecc-actions | ||
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | ||
| env: | ||
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | ||
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | ||
| # - name: Delete readonly role for scans | ||
| # uses: ./ecc-actions/auto-test-actions/readonly-role-for-scans | ||
| # with: | ||
| # CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | ||
| # SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | ||
| # WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | ||
| # COMPLIANCE: ${{ matrix.compliance }} | ||
| # PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }} | ||
| # ROLE_ACTION: "delete" | ||
| # env: | ||
| # created_role_name: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | ||
| destroy_common_resources: | ||
| name: Destroy common | ||
| runs-on: ubuntu-22.04 | ||
| needs: [delete_readonly_role_for_scans] | ||
| if: ${{ always() }} | ||
| strategy: | ||
| max-parallel: 10 | ||
| fail-fast: false | ||
| matrix: | ||
| compliance: ["green", "red"] | ||
| env: | ||
| COMPLINCE: ${{ matrix.compliance }} | ||
| RESOURCE: common_resources | ||
| steps: | ||
| - name: Git clone the repository | ||
| uses: actions/checkout@v4 | ||
| - name: Checkout ecc-actions | ||
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | ||
| env: | ||
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | ||
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | ||
| - name: Destroy common resources | ||
| uses: ./ecc-actions/auto-test-actions/destroy-common-resources | ||
| with: | ||
| CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | ||
| SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | ||
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | ||
| COMPLIANCE: ${{ matrix.compliance }} | ||
| check_resources: | ||
| name: Check left-over resources | ||
| runs-on: ubuntu-22.04 | ||
| needs: [destroy_common_resources] | ||
| if: ${{ always() }} | ||
| steps: | ||
| - name: Git clone the repository | ||
| uses: actions/checkout@v4 | ||
| - name: Checkout ecc-actions | ||
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | ||
| env: | ||
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | ||
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | ||
| - name: Check left-over resources | ||
| uses: ./ecc-actions/auto-test-actions/check-resources | ||
| with: | ||
| CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | ||
| SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | ||
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | ||
| COMPLIANCE: ${{ matrix.compliance }} | ||
