skip: add sequential parallel resources #258
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Auto-testing | |
| on: | |
| push: | |
| branches: | |
| - "feature/policy_testing_v2" | |
| - "feature/deploy_and_scan_sequential_parallel_resources" | |
| # Run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| inputs: | |
| resource_priority_list: | |
| type: string | |
| description: Priority list for resources (you can remove unnecessary resources during testing) | |
| default: '[ ]' | |
| required: true | |
| # limits to only one workflow in time | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| env: | |
| AUTO_TEST_DIR: "auto_policy_testing" | |
| TF_VAR_project: ${{ secrets.TF_VAR_project }} | |
| TF_VAR_region: ${{ secrets.AWS_REGION }} | |
| TF_VAR_zone: ${{ secrets.TF_VAR_zone }} | |
| TF_BACKEND_STORAGE_NAME: ${{ secrets.TF_BACKEND_STORAGE_NAME }} | |
| TF_CLI_ARGS: "-no-color" | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} | |
| default_resource_priority_list: "['account', 'glue', 'sns']" | |
| #'[ "account", "acm", "airflow", "ami", "apigwv2", "app-elb", "app-flow", "asg", "backup", "cfn", "cloudtrail", "codebuild", "codedeploy", "codepipeline", "dax", "directory", "distribution", "dlm", "dms", "dynamodb", "ebs", "ec2", "ecr", "ecs", "efs", "eip", "eks", "elasticache", "elasticbeanstalk", "elasticsearch", "elb", "emr", "eni", "event", "firehose", "fsx", "glacier", "glue", "graphql", "hostedzone", "iam", "internet", "kafka", "key", "kinesis", "kms", "lambda", "launch", "lightsail", "log", "message", "nat", "network", "peering", "r53domain", "rds", "redshift", "rest", "route", "rrset", "s3", "sagemaker", "secrets", "security-group", "sns", "sqs", "step", "subnet", "transit", "vpc", "vpn", "waf"]' | |
| RED: '\033[0;31m' | |
| ACTIONS_REPO_BRANCH: "feature/deploy_and_scan_sequential_parallel_resources" | |
| permissions: | |
| contents: "read" | |
| id-token: "write" | |
| jobs: | |
| deploy_common_resources: | |
| name: Deploy common | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| compliance: ["green", "red"] | |
| env: | |
| COMPLINCE: ${{ matrix.compliance }} | |
| RESOURCE: common_resources | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout ecc-actions | |
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | |
| env: | |
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | |
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | |
| - name: Deploy common | |
| uses: ./ecc-actions/auto-test-actions/deploy-common-resources | |
| with: | |
| CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | |
| SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | |
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| COMPLIANCE: ${{ matrix.compliance }} | |
| create_readonly_role_for_scans: | |
| name: Create readonly role for scans | |
| if: github.repository == 'epam/ecc-aws-rulepack' | |
| runs-on: ubuntu-22.04 | |
| needs: deploy_common_resources | |
| outputs: | |
| readonly_role_name: ${{ steps.create-readonly-role.outputs.readonly_role_name }} | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout ecc-actions | |
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | |
| env: | |
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | |
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | |
| - name: Create readonly role for scans | |
| id: create-readonly-role | |
| uses: ./ecc-actions/auto-test-actions/readonly-role-for-scans | |
| with: | |
| CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | |
| SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | |
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| COMPLIANCE: ${{ matrix.compliance }} | |
| PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }} | |
| ROLE_ACTION: "create" | |
| # pack_iam_policies_per_resource: | |
| # runs-on: ubuntu-22.04 | |
| # needs: deploy_common_resources | |
| # steps: | |
| # - name: Git clone the repository | |
| # uses: actions/checkout@v4 | |
| # - name: pack_iam_policies | |
| # shell: bash | |
| # working-directory: ./ecc-actions/auto-test-actions/scripts | |
| # run: python -u pack_iam.py ${{ github.repository }} | |
| # - name: Archive loggs to artifacts | |
| # uses: actions/upload-artifact@v4 | |
| # with: | |
| # name: pack_iam | |
| # path: | | |
| # ${{ env.AUTO_TEST_DIR }}/iam | |
| prepare_resource_matrix: | |
| name: Prepare resource matrix | |
| runs-on: ubuntu-22.04 | |
| needs: deploy_common_resources | |
| outputs: | |
| action_repo_name: ${{ steps.prepare-resource-matrix.outputs.action_repo_name }} | |
| parallel_resources_list: ${{ steps.prepare-resource-matrix.outputs.parallel_resources_to_scan }} | |
| not_parallel_resources_list: ${{ steps.prepare-resource-matrix.outputs.not_parallel_resources_to_scan }} | |
| sequential_resources_list: ${{ steps.prepare-resource-matrix.outputs.sequential_resources_to_scan }} | |
| sequential_resources_length: ${{ steps.prepare-resource-matrix.outputs.sequential_resources_length }} | |
| sequential_parallel_resources_list: ${{ steps.prepare-resource-matrix.outputs.sequential_parallel_resources_to_scan }} | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout ecc-actions | |
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | |
| env: | |
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | |
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | |
| - name: Prepare resource matrix | |
| id: prepare-resource-matrix | |
| uses: ./ecc-actions/auto-test-actions/prepare-resource-matrix | |
| env: | |
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | |
| deploy_and_scan_parallel_resources: | |
| name: Scan P | |
| runs-on: ubuntu-22.04 | |
| needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] # pack_iam_policies_per_resource, | |
| if: ${{ needs.prepare_resource_matrix.outputs.parallel_resources_list != '[]' }} | |
| strategy: | |
| max-parallel: 10 | |
| fail-fast: false | |
| matrix: | |
| compliance: ['green', 'red'] | |
| resource: ${{fromJson(needs.prepare_resource_matrix.outputs.parallel_resources_list)}} | |
| env: | |
| COMPLINCE: ${{ matrix.compliance }} | |
| RESOURCE: ${{ matrix.resource }} | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout ecc-actions | |
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | |
| env: | |
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | |
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | |
| - name: Deploy and scan parallel resources | |
| uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources | |
| with: | |
| CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | |
| SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | |
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| COMPLIANCE: ${{ matrix.compliance }} | |
| PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }} | |
| READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | |
| GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }} | |
| deploy_and_scan_not_parallel_resources: | |
| name: Scan N/P | |
| runs-on: ubuntu-22.04 | |
| needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] # pack_iam_policies_per_resource, | |
| if: ${{ needs.prepare_resource_matrix.outputs.not_parallel_resources_list != '[]' }} | |
| strategy: | |
| max-parallel: 1 | |
| fail-fast: false | |
| matrix: | |
| compliance: ['green', 'red'] | |
| resource: ${{fromJson(needs.prepare_resource_matrix.outputs.not_parallel_resources_list)}} | |
| env: | |
| COMPLINCE: ${{ matrix.compliance }} | |
| RESOURCE: ${{ matrix.resource }} | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout ecc-actions | |
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | |
| env: | |
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | |
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | |
| - name: Deploy and scan non-parallel resources | |
| uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources | |
| with: | |
| CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | |
| SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | |
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| COMPLIANCE: ${{ matrix.compliance }} | |
| PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }} | |
| READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | |
| GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }} | |
| deploy_and_scan_sequential_resources: | |
| name: Scan S | |
| runs-on: ubuntu-22.04 | |
| needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] | |
| if: ${{ needs.prepare_resource_matrix.outputs.sequential_resources_list != '[]' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| resource: ${{fromJson(needs.prepare_resource_matrix.outputs.sequential_resources_list)}} | |
| env: | |
| RESOURCE: ${{ matrix.resource }} | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout ecc-actions | |
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | |
| env: | |
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | |
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | |
| - name: Deploy and scan non-parallel resource (green) | |
| uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources | |
| env: | |
| COMPLINCE: "green" | |
| with: | |
| CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | |
| SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | |
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| COMPLIANCE: ${{ matrix.compliance }} | |
| PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }} | |
| READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | |
| GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }} | |
| - name: Deploy and scan non-parallel resource (red) | |
| uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources | |
| env: | |
| COMPLINCE: "red" | |
| if: always() | |
| with: | |
| CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | |
| SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | |
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| COMPLIANCE: ${{ matrix.compliance }} | |
| PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }} | |
| READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | |
| GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }} | |
| deploy_and_scan_sequential_parallel_resources: | |
| name: Scan S/P | |
| needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] | |
| strategy: | |
| max-parallel: 1 | |
| fail-fast: false | |
| matrix: | |
| resource: ['glue'] | |
| # resource: ${{fromJson(needs.prepare_resource_matrix.outputs.sequential_parallel_resources_list)}} | |
| uses: ./.github/workflows/sequential_parallel_scan.yml | |
| secrets: inherit | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.run_id }}-sequential-parallel | |
| with: | |
| RESOURCE: ${{ matrix.resource }} | |
| ACTIONS_REPO_BRANCH: ${{ needs.prepare_resource_matrix.outputs.action_repo_name }} | |
| READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | |
| delete_readonly_role_for_scans: | |
| name: Delete readonly role for scans | |
| if: ${{ always() }} | |
| runs-on: ubuntu-22.04 | |
| needs: [ create_readonly_role_for_scans, deploy_and_scan_parallel_resources, deploy_and_scan_not_parallel_resources, deploy_and_scan_sequential_resources, deploy_and_scan_sequential_parallel_resources ] | |
| # needs: [ create_readonly_role_for_scans, deploy_and_scan_sequential_parallel_resources ] | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout ecc-actions | |
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | |
| env: | |
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | |
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | |
| # - name: Delete readonly role for scans | |
| # uses: ./ecc-actions/auto-test-actions/readonly-role-for-scans | |
| # with: | |
| # CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | |
| # SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | |
| # WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| # COMPLIANCE: ${{ matrix.compliance }} | |
| # PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }} | |
| # ROLE_ACTION: "delete" | |
| # env: | |
| # created_role_name: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | |
| destroy_common_resources: | |
| name: Destroy common | |
| runs-on: ubuntu-22.04 | |
| needs: [delete_readonly_role_for_scans] | |
| if: ${{ always() }} | |
| strategy: | |
| max-parallel: 10 | |
| fail-fast: false | |
| matrix: | |
| compliance: ["green", "red"] | |
| env: | |
| COMPLINCE: ${{ matrix.compliance }} | |
| RESOURCE: common_resources | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout ecc-actions | |
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | |
| env: | |
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | |
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | |
| - name: Destroy common resources | |
| uses: ./ecc-actions/auto-test-actions/destroy-common-resources | |
| with: | |
| CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | |
| SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | |
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| COMPLIANCE: ${{ matrix.compliance }} | |
| check_resources: | |
| name: Check left-over resources | |
| runs-on: ubuntu-22.04 | |
| needs: [destroy_common_resources] | |
| if: ${{ always() }} | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout ecc-actions | |
| run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions | |
| env: | |
| PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }} | |
| ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }} | |
| - name: Check left-over resources | |
| uses: ./ecc-actions/auto-test-actions/check-resources | |
| with: | |
| CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }} | |
| SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }} | |
| WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| COMPLIANCE: ${{ matrix.compliance }} |