skip: add auto-test for codebuild #102
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Auto-testing | |
| on: | |
| push: | |
| branches: | |
| - 'feature/policy_testing' | |
| - 'feature/policy_testing_v2' | |
| # Run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| inputs: | |
| resource_priority_list: | |
| type: string | |
| description: Priority list for resources (you can remove unnecessary resources during testing) | |
| default: '[ "security-group"]' | |
| required: true | |
| # limits to only one workflow in time | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| env: | |
| AUTO_TEST_DIR: "auto_policy_testing" | |
| TF_VAR_project: ${{ secrets.TF_VAR_project }} | |
| TF_VAR_region: ${{ secrets.AWS_REGION }} | |
| TF_VAR_zone: ${{ secrets.TF_VAR_zone }} | |
| TF_BACKEND_STORAGE_NAME: ${{ secrets.TF_BACKEND_STORAGE_NAME }} | |
| TF_CLI_ARGS: "-no-color" | |
| AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} | |
| default_resource_priority_list: '[ "codebuild"]' | |
| RED: '\033[0;31m' | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| jobs: | |
| deploy_common_resources: | |
| name: Deploy common | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| compliance: ['green', 'red'] | |
| env: | |
| COMPLINCE: ${{ matrix.compliance }} | |
| RESOURCE: common_resources | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - uses: ./.github/workflows/actions/set-up-cloud-creds | |
| with: | |
| aws_assume_role: ${{ secrets.CI_ASSUME_ROLE }} | |
| service_account: ${{ secrets.SERVICE_ACCOUNT }} | |
| workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| step: common_${{ matrix.compliance }} | |
| - name: Clean old S3 logs | |
| shell: bash | |
| run: | | |
| cloud=$(echo "${{ github.repository }}" | cut -d'-' -f 2) | |
| aws s3 rm s3://$TF_BACKEND_STORAGE_NAME/$cloud/logs --recursive | |
| aws s3 rm s3://$TF_BACKEND_STORAGE_NAME/$cloud/policy_output --recursive | |
| - name: Deploy common resources | |
| uses: ./.github/workflows/actions/tf-init-apply | |
| - uses: ./.github/workflows/actions/publish-logs-to-s3 | |
| if: ${{ always() }} | |
| with: | |
| local_path: "${{ github.workspace }}/${{ env.AUTO_TEST_DIR }}/logs" | |
| remote_path: "logs" | |
| create_readonly_role_for_scans: | |
| if: github.repository == 'epam/ecc-aws-rulepack' | |
| runs-on: ubuntu-22.04 | |
| needs: deploy_common_resources | |
| outputs: | |
| readonly_role_name: ${{ steps.create-AWS-readonly-role.outputs.readonly_role_name }} | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - uses: ./.github/workflows/actions/set-up-cloud-creds | |
| with: | |
| aws_assume_role: ${{ secrets.CI_ASSUME_ROLE }} | |
| service_account: ${{ secrets.SERVICE_ACCOUNT }} | |
| workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| step: common_${{ matrix.compliance }} | |
| - uses: ./.github/workflows/actions/install-dependencies | |
| - name: create AWS readonly role | |
| id: create-AWS-readonly-role | |
| shell: bash | |
| working-directory: ${{ env.AUTO_TEST_DIR }}/scripts | |
| run: | | |
| source .venv/bin/activate | |
| if [ ${{ github.repository }} == 'epam/ecc-aws-rulepack' ]; then | |
| random_number=$((RANDOM % 9000 + 1000)) | |
| readonly_role_name="github-ci-readonly-ecc-rulepack-$random_number" | |
| echo "readonly_role_name=$readonly_role_name" >> "$GITHUB_OUTPUT" | |
| python aws_iam_readonly_role.py \ | |
| --ci_exec_role_name ${{ secrets.CI_ASSUME_ROLE }} \ | |
| --ci_readonly_role_name $readonly_role_name\ | |
| --create --role | |
| elif [ ${{ github.repository }} == 'epam/ecc-gcp-rulepack' ]; then | |
| echo 'TO DO' | |
| elif [ ${{ github.repository }} == 'epam/ecc-azure-rulepack' ]; then | |
| echo 'TO DO' | |
| fi | |
| pack_iam_policies_per_resource: | |
| runs-on: ubuntu-22.04 | |
| needs: deploy_common_resources | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - name: pack_iam_policies | |
| shell: bash | |
| working-directory: ${{ env.AUTO_TEST_DIR }} | |
| run: | | |
| cd scripts | |
| python pack_iam.py ${{ github.repository }} | |
| - name: Archive loggs to artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pack_iam | |
| path: | | |
| ${{ env.AUTO_TEST_DIR }}/iam | |
| prepare_matrix_resources: | |
| runs-on: ubuntu-22.04 | |
| needs: [ deploy_common_resources ] | |
| outputs: | |
| parallel_resources_list: ${{ steps.pass-resource-priority-list-to-matrix.outputs.parallel_resources_to_scan }} | |
| not_parallel_resources_list: ${{ steps.pass-resource-priority-list-to-matrix.outputs.not_parallel_resources_to_scan }} | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - id: pass-resource-priority-list-to-matrix | |
| shell: bash | |
| working-directory: ${{ env.AUTO_TEST_DIR }}/scripts | |
| run: | | |
| if [ "${{ inputs.resource_priority_list }}" != '' ]; then | |
| resource_priority_list="${{ inputs.resource_priority_list }}" | |
| else | |
| resource_priority_list="$default_resource_priority_list" | |
| fi | |
| cloud=$(echo "${{ github.repository }}" | cut -d'-' -f 2) | |
| not_parallel_resources="$(python -c "import exception_rules; print(exception_rules.$cloud.get('not-parallel',[]))")" | |
| source resources_to_scan.sh | |
| parallel_resources_to_scan="$(get_parallel_resources_to_scan "$not_parallel_resources" "$resource_priority_list")" | |
| not_parallel_resources_to_scan="$(get_not_parallel_resources_to_scan "$not_parallel_resources" "$resource_priority_list")" | |
| echo "parallel_resources_to_scan=$parallel_resources_to_scan" | |
| echo "not_parallel_resources_to_scan=$not_parallel_resources_to_scan" | |
| echo "parallel_resources_to_scan=$parallel_resources_to_scan" >> "$GITHUB_OUTPUT" | |
| echo "not_parallel_resources_to_scan=$not_parallel_resources_to_scan" >> "$GITHUB_OUTPUT" | |
| deploy_and_scan_parallel_resources: | |
| name: Scan P | |
| runs-on: ubuntu-22.04 | |
| needs: [ deploy_common_resources, pack_iam_policies_per_resource, create_readonly_role_for_scans, prepare_matrix_resources] | |
| strategy: | |
| max-parallel: 10 | |
| fail-fast: false | |
| matrix: | |
| compliance: ['green', 'red'] | |
| resource: ${{fromJson(needs.prepare_matrix_resources.outputs.parallel_resources_list)}} | |
| env: | |
| COMPLINCE: ${{ matrix.compliance }} | |
| RESOURCE: ${{ matrix.resource }} | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - uses: ./.github/workflows/actions/set-up-cloud-creds | |
| with: | |
| aws_assume_role: ${{ secrets.CI_ASSUME_ROLE }} | |
| service_account: ${{ secrets.SERVICE_ACCOUNT }} | |
| workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| step: common_${{ matrix.compliance }} | |
| - uses: ./.github/workflows/actions/install-dependencies | |
| - name: Download a single artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: pack_iam | |
| path: ${{ env.AUTO_TEST_DIR }}/iam | |
| - name: Deploy terraform resources | |
| uses: ./.github/workflows/actions/tf-init-apply | |
| - name: Custodian scan | |
| uses: ./.github/workflows/actions/c7n-scan | |
| with: | |
| readonly_role_name: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | |
| GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }} | |
| - name: Destroy terraform resources | |
| uses: ./.github/workflows/actions/tf-destroy | |
| if: ${{ always() }} | |
| - uses: ./.github/workflows/actions/publish-logs-to-s3 | |
| if: ${{ always() }} | |
| with: | |
| local_path: "${{ github.workspace }}/${{ env.AUTO_TEST_DIR }}/logs" | |
| remote_path: "logs" | |
| - uses: ./.github/workflows/actions/publish-logs-to-s3 | |
| if: ${{ always() }} | |
| with: | |
| local_path: "${{ github.workspace }}/${{ env.AUTO_TEST_DIR }}/output" | |
| remote_path: "policy_output/${{ matrix.compliance }}" | |
| deploy_and_scan_not_parallel_resources: | |
| name: Scan N/P | |
| runs-on: ubuntu-22.04 | |
| needs: [ deploy_common_resources, pack_iam_policies_per_resource, create_readonly_role_for_scans, prepare_matrix_resources] | |
| if: ${{ needs.prepare_matrix_resources.outputs.not_parallel_resources_list != '[]' }} | |
| strategy: | |
| max-parallel: 1 | |
| fail-fast: false | |
| matrix: | |
| compliance: ['green', 'red'] | |
| resource: ${{fromJson(needs.prepare_matrix_resources.outputs.not_parallel_resources_list)}} | |
| env: | |
| COMPLINCE: ${{ matrix.compliance }} | |
| RESOURCE: ${{ matrix.resource }} | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - uses: ./.github/workflows/actions/set-up-cloud-creds | |
| with: | |
| aws_assume_role: ${{ secrets.CI_ASSUME_ROLE }} | |
| service_account: ${{ secrets.SERVICE_ACCOUNT }} | |
| workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| step: common_${{ matrix.compliance }} | |
| - uses: ./.github/workflows/actions/install-dependencies | |
| - name: Download a single artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: pack_iam | |
| path: ${{ env.AUTO_TEST_DIR }}/iam | |
| - name: Deploy terraform resources | |
| uses: ./.github/workflows/actions/tf-init-apply | |
| - name: Custodian scan | |
| uses: ./.github/workflows/actions/c7n-scan | |
| with: | |
| readonly_role_name: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }} | |
| GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }} | |
| - name: Destroy terraform resources | |
| uses: ./.github/workflows/actions/tf-destroy | |
| if: ${{ always() }} | |
| - uses: ./.github/workflows/actions/publish-logs-to-s3 | |
| if: ${{ always() }} | |
| with: | |
| local_path: "${{ github.workspace }}/${{ env.AUTO_TEST_DIR }}/logs" | |
| remote_path: "logs" | |
| - uses: ./.github/workflows/actions/publish-logs-to-s3 | |
| if: ${{ always() }} | |
| with: | |
| local_path: "${{ github.workspace }}/${{ env.AUTO_TEST_DIR }}/output" | |
| remote_path: "policy_output/${{ matrix.compliance }}" | |
| remove_readonly_role_for_scans: | |
| if: ${{ always() }} | |
| runs-on: ubuntu-22.04 | |
| needs: [deploy_and_scan_parallel_resources, create_readonly_role_for_scans, deploy_and_scan_not_parallel_resources] | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - uses: ./.github/workflows/actions/set-up-cloud-creds | |
| with: | |
| aws_assume_role: ${{ secrets.CI_ASSUME_ROLE }} | |
| service_account: ${{ secrets.SERVICE_ACCOUNT }} | |
| workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| step: common_${{ matrix.compliance }} | |
| - uses: ./.github/workflows/actions/install-dependencies | |
| - name: remove-AWS-readonly-role | |
| shell: bash | |
| if: ${{ github.repository == 'epam/ecc-aws-rulepack' && always() }} | |
| working-directory: ${{ env.AUTO_TEST_DIR }}/scripts | |
| run: | | |
| source .venv/bin/activate | |
| if [ ${{ github.repository }} == 'epam/ecc-aws-rulepack' ]; then | |
| readonly_role_name="${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}" | |
| echo $readonly_role_name | |
| python aws_iam_readonly_role.py \ | |
| --ci_exec_role_name ${{ secrets.CI_ASSUME_ROLE }} \ | |
| --ci_readonly_role_name "$readonly_role_name"\ | |
| --delete --role | |
| fi | |
| destroy_common_resources: | |
| name: Destroy common | |
| runs-on: ubuntu-22.04 | |
| needs: [deploy_common_resources, deploy_and_scan_parallel_resources,deploy_and_scan_not_parallel_resources] | |
| if: ${{ always() }} | |
| strategy: | |
| max-parallel: 10 | |
| fail-fast: false | |
| matrix: | |
| compliance: ['green', 'red'] | |
| env: | |
| COMPLINCE: ${{ matrix.compliance }} | |
| RESOURCE: common_resources | |
| steps: | |
| - name: Git clone the repository | |
| uses: actions/checkout@v4 | |
| - uses: ./.github/workflows/actions/set-up-cloud-creds | |
| with: | |
| aws_assume_role: ${{ secrets.CI_ASSUME_ROLE }} | |
| service_account: ${{ secrets.SERVICE_ACCOUNT }} | |
| workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }} | |
| step: common_${{ matrix.compliance }} | |
| - name: Destroy common resources | |
| if: ${{ always() }} | |
| uses: ./.github/workflows/actions/tf-destroy | |
| - uses: ./.github/workflows/actions/publish-logs-to-s3 | |
| if: ${{ always() }} | |
| with: | |
| local_path: "${{ github.workspace }}/${{ env.AUTO_TEST_DIR }}/logs" | |
| remote_path: "logs" |