Skip to content

Commit

Permalink
skip: add sequential parallel resources
Browse files Browse the repository at this point in the history
  • Loading branch information
Mykhailo committed Oct 4, 2024
1 parent 4befa6d commit 9cfbbb8
Show file tree
Hide file tree
Showing 2 changed files with 189 additions and 109 deletions.
242 changes: 133 additions & 109 deletions .github/workflows/auto-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ on:
push:
branches:
- "feature/policy_testing_v2"
- "feature/deploy_and_scan_sequential_parallel_resources"
# Run this workflow manually from the Actions tab
workflow_dispatch:
inputs:
Expand All @@ -14,7 +15,7 @@ on:

# limits to only one workflow in time
concurrency:
group: ${{ github.workflow }}
group: ${{ github.workflow }}-${{ github.ref }}

env:
AUTO_TEST_DIR: "auto_policy_testing"
Expand All @@ -27,7 +28,7 @@ env:
default_resource_priority_list: "['account', 'glue', 'sns']"
#'[ "account", "acm", "airflow", "ami", "apigwv2", "app-elb", "app-flow", "asg", "backup", "cfn", "cloudtrail", "codebuild", "codedeploy", "codepipeline", "dax", "directory", "distribution", "dlm", "dms", "dynamodb", "ebs", "ec2", "ecr", "ecs", "efs", "eip", "eks", "elasticache", "elasticbeanstalk", "elasticsearch", "elb", "emr", "eni", "event", "firehose", "fsx", "glacier", "glue", "graphql", "hostedzone", "iam", "internet", "kafka", "key", "kinesis", "kms", "lambda", "launch", "lightsail", "log", "message", "nat", "network", "peering", "r53domain", "rds", "redshift", "rest", "route", "rrset", "s3", "sagemaker", "secrets", "security-group", "sns", "sqs", "step", "subnet", "transit", "vpc", "vpn", "waf"]'
RED: '\033[0;31m'
ACTIONS_REPO_BRANCH: "main"
ACTIONS_REPO_BRANCH: "feature/deploy_and_scan_sequential_parallel_resources"

permissions:
contents: "read"
Expand Down Expand Up @@ -112,10 +113,12 @@ jobs:
runs-on: ubuntu-22.04
needs: deploy_common_resources
outputs:
action_repo_name: ${{ steps.prepare-resource-matrix.outputs.action_repo_name }}
parallel_resources_list: ${{ steps.prepare-resource-matrix.outputs.parallel_resources_to_scan }}
not_parallel_resources_list: ${{ steps.prepare-resource-matrix.outputs.not_parallel_resources_to_scan }}
sequential_resources_list: ${{ steps.prepare-resource-matrix.outputs.sequential_resources_to_scan }}
sequential_resources_length: ${{ steps.prepare-resource-matrix.outputs.sequential_resources_length }}
sequential_parallel_resources_list: ${{ steps.prepare-resource-matrix.outputs.sequential_parallel_resources_to_scan }}
steps:
- name: Git clone the repository
uses: actions/checkout@v4
Expand All @@ -129,129 +132,150 @@ jobs:
- name: Prepare resource matrix
id: prepare-resource-matrix
uses: ./ecc-actions/auto-test-actions/prepare-resource-matrix

deploy_and_scan_parallel_resources:
name: Scan P
runs-on: ubuntu-22.04
needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] # pack_iam_policies_per_resource,
if: ${{ needs.prepare_resource_matrix.outputs.parallel_resources_list != '[]' }}
strategy:
max-parallel: 10
fail-fast: false
matrix:
compliance: ['green', 'red']
resource: ${{fromJson(needs.prepare_resource_matrix.outputs.parallel_resources_list)}}
env:
COMPLINCE: ${{ matrix.compliance }}
RESOURCE: ${{ matrix.resource }}
steps:
- name: Git clone the repository
uses: actions/checkout@v4

- name: Checkout ecc-actions
run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
env:
PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }}

# deploy_and_scan_parallel_resources:
# name: Scan P
# runs-on: ubuntu-22.04
# needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] # pack_iam_policies_per_resource,
# if: ${{ needs.prepare_resource_matrix.outputs.parallel_resources_list != '[]' }}
# strategy:
# max-parallel: 10
# fail-fast: false
# matrix:
# compliance: ['green', 'red']
# resource: ${{fromJson(needs.prepare_resource_matrix.outputs.parallel_resources_list)}}
# env:
# COMPLINCE: ${{ matrix.compliance }}
# RESOURCE: ${{ matrix.resource }}
# steps:
# - name: Git clone the repository
# uses: actions/checkout@v4

# - name: Checkout ecc-actions
# run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
# env:
# PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
# ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }}

- name: Deploy and scan parallel resources
uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
with:
CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
COMPLIANCE: ${{ matrix.compliance }}
PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}
# - name: Deploy and scan parallel resources
# uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
# with:
# CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
# SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
# WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
# COMPLIANCE: ${{ matrix.compliance }}
# PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
# READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
# GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}

deploy_and_scan_not_parallel_resources:
name: Scan N/P
runs-on: ubuntu-22.04
needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] # pack_iam_policies_per_resource,
if: ${{ needs.prepare_resource_matrix.outputs.not_parallel_resources_list != '[]' }}
strategy:
max-parallel: 1
fail-fast: false
matrix:
compliance: ['green', 'red']
resource: ${{fromJson(needs.prepare_resource_matrix.outputs.not_parallel_resources_list)}}
env:
COMPLINCE: ${{ matrix.compliance }}
RESOURCE: ${{ matrix.resource }}
steps:
- name: Git clone the repository
uses: actions/checkout@v4
# deploy_and_scan_not_parallel_resources:
# name: Scan N/P
# runs-on: ubuntu-22.04
# needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix] # pack_iam_policies_per_resource,
# if: ${{ needs.prepare_resource_matrix.outputs.not_parallel_resources_list != '[]' }}
# strategy:
# max-parallel: 1
# fail-fast: false
# matrix:
# compliance: ['green', 'red']
# resource: ${{fromJson(needs.prepare_resource_matrix.outputs.not_parallel_resources_list)}}
# env:
# COMPLINCE: ${{ matrix.compliance }}
# RESOURCE: ${{ matrix.resource }}
# steps:
# - name: Git clone the repository
# uses: actions/checkout@v4

- name: Checkout ecc-actions
run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
env:
PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }}
- name: Deploy and scan non-parallel resources
uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
with:
CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
COMPLIANCE: ${{ matrix.compliance }}
PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}
# - name: Checkout ecc-actions
# run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
# env:
# PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
# ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }}
# - name: Deploy and scan non-parallel resources
# uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
# with:
# CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
# SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
# WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
# COMPLIANCE: ${{ matrix.compliance }}
# PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
# READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
# GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}

deploy_and_scan_sequential_resources:
name: Scan S
runs-on: ubuntu-22.04
# deploy_and_scan_sequential_resources:
# name: Scan S
# runs-on: ubuntu-22.04
# needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix]
# if: ${{ needs.prepare_resource_matrix.outputs.sequential_resources_list != '[]' }}
# strategy:
# fail-fast: false
# matrix:
# resource: ${{fromJson(needs.prepare_resource_matrix.outputs.sequential_resources_list)}}
# env:
# RESOURCE: ${{ matrix.resource }}
# steps:
# - name: Git clone the repository
# uses: actions/checkout@v4

# - name: Checkout ecc-actions
# run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
# env:
# PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
# ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }}

# - name: Deploy and scan non-parallel resource (green)
# uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
# env:
# COMPLINCE: "green"
# with:
# CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
# SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
# WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
# COMPLIANCE: ${{ matrix.compliance }}
# PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
# READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
# GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}

# - name: Deploy and scan non-parallel resource (red)
# uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
# env:
# COMPLINCE: "red"
# if: always()
# with:
# CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
# SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
# WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
# COMPLIANCE: ${{ matrix.compliance }}
# PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
# READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
# GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}

deploy_and_scan_sequential_parallel_resources:
name: Scan S/P
needs: [deploy_common_resources, create_readonly_role_for_scans, prepare_resource_matrix]
if: ${{ needs.prepare_resource_matrix.outputs.sequential_resources_list != '[]' }}
strategy:
max-parallel: 1
fail-fast: false
matrix:
resource: ${{fromJson(needs.prepare_resource_matrix.outputs.sequential_resources_list)}}
env:
resource: ['glue']
# resource: ${{fromJson(needs.prepare_resource_matrix.outputs.sequential_parallel_resources_list)}}
uses: ./.github/workflows/sequential_parallel_scan.yml
secrets: inherit
concurrency:
group: ${{ github.workflow }}-${{ github.run_id }}-sequential-parallel
with:
RESOURCE: ${{ matrix.resource }}
steps:
- name: Git clone the repository
uses: actions/checkout@v4

- name: Checkout ecc-actions
run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
env:
PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
ACTIONS_REPO_BRANCH: ${{ env.ACTIONS_REPO_BRANCH }}

- name: Deploy and scan non-parallel resource (green)
uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
env:
COMPLINCE: "green"
with:
CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
COMPLIANCE: ${{ matrix.compliance }}
PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}

- name: Deploy and scan non-parallel resource (red)
uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
env:
COMPLINCE: "red"
if: always()
with:
CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
COMPLIANCE: ${{ matrix.compliance }}
PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}
GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}
ACTIONS_REPO_BRANCH: ${{ needs.prepare_resource_matrix.outputs.action_repo_name }}
READONLY_ROLE_NAME: ${{ needs.create_readonly_role_for_scans.outputs.readonly_role_name }}

delete_readonly_role_for_scans:
name: Delete readonly role for scans
if: ${{ always() }}
runs-on: ubuntu-22.04
needs: [ create_readonly_role_for_scans, deploy_and_scan_parallel_resources, deploy_and_scan_not_parallel_resources, deploy_and_scan_sequential_resources ]
# needs: [ create_readonly_role_for_scans, deploy_and_scan_parallel_resources, deploy_and_scan_not_parallel_resources, deploy_and_scan_sequential_resources, deploy_and_scan_sequential_parallel_resources ]
needs: [ create_readonly_role_for_scans, deploy_and_scan_sequential_parallel_resources ]
steps:
- name: Git clone the repository
uses: actions/checkout@v4
Expand Down
56 changes: 56 additions & 0 deletions .github/workflows/sequential_parallel_scan.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
name: Deploy and scan sequential parallel resources

on:
workflow_call:
inputs:
RESOURCE:
type: string
ACTIONS_REPO_BRANCH:
type: string
READONLY_ROLE_NAME:
type: string

env:
AUTO_TEST_DIR: "auto_policy_testing"
TF_VAR_project: ${{ secrets.TF_VAR_project }}
TF_VAR_region: ${{ secrets.AWS_REGION }}
TF_VAR_zone: ${{ secrets.TF_VAR_zone }}
TF_BACKEND_STORAGE_NAME: ${{ secrets.TF_BACKEND_STORAGE_NAME }}
TF_CLI_ARGS: "-no-color"
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}

jobs:
deploy_and_scan_sequential_parallel_resources:
name: Scan S/P
runs-on: ubuntu-22.04
# concurrency:
# group: ${{ github.workflow }}-${{ github.run_id }}-nested
strategy:
max-parallel: 2
fail-fast: false
matrix:
compliance: ['green', 'red']
env:
COMPLINCE: ${{ matrix.compliance }}
steps:
- name: Git clone the repository
uses: actions/checkout@v4

- name: Checkout ecc-actions
run: git clone -b $ACTIONS_REPO_BRANCH "https://git:$PROJECT_TOKEN@git.epam.com/epmc-sec/cloudlab/cloud_custodian/ecc-actions.git" ecc-actions
env:
PROJECT_TOKEN: ${{ secrets.ECC_CHANGELOG_ACTION }}
ACTIONS_REPO_BRANCH: ${{ inputs.ACTIONS_REPO_BRANCH }}

- name: Deploy and scan sequential parallel resources
uses: ./ecc-actions/auto-test-actions/deploy-and-scan-resources
with:
CI_ASSUME_ROLE: ${{ secrets.CI_ASSUME_ROLE }}
SERVICE_ACCOUNT: ${{ secrets.SERVICE_ACCOUNT }}
WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.WORKLOAD_IDENTITY_PROVIDER }}
COMPLIANCE: ${{ matrix.compliance }}
PROJECT_TOKEN: ${{ secrets.CLOUDCUSTODIAN_CORE }}
READONLY_ROLE_NAME: ${{ inputs.READONLY_ROLE_NAME }}
GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_IMPERSONATE_SERVICE_ACCOUNT }}


0 comments on commit 9cfbbb8

Please sign in to comment.