From 2c8b4bee7ee562575bcd4b40dfb8c49578360a8d Mon Sep 17 00:00:00 2001 From: Mohamed Nur <85885638+mohamed-cds@users.noreply.github.com> Date: Tue, 12 Jul 2022 14:30:00 -0400 Subject: [PATCH] feat: staging environment (#182) * feat: staging environment * chore: use right account id * chore: remove scan queue for later * chore: remove from matrix * chore: apply workflow * fix: use correct secret * chore: remove condition * chore: override defaults for integration test module * chore: typo * chore: bump and removed unused input --- .github/workflows/tf_apply_staging.yml | 93 +++++++ .github/workflows/tf_plan_staging.yml | 60 +++++ terragrunt/aws/api/inputs.tf | 4 + terragrunt/aws/api/oidc_iam.tf | 2 +- .../aws/integration_test/s3_scan_object.tf | 9 +- terragrunt/env/backend.tf | 10 - terragrunt/env/production/api/terragrunt.hcl | 1 + terragrunt/env/staging/alarms/terragrunt.hcl | 25 ++ .../env/staging/api/.terraform.lock.hcl | 60 +++++ terragrunt/env/staging/api/terragrunt.hcl | 27 ++ terragrunt/env/staging/env_vars.hcl | 6 + .../staging/hosted_zone/.terraform.lock.hcl | 22 ++ .../env/staging/hosted_zone/terragrunt.hcl | 7 + .../staging/integration_test/terragrunt.hcl | 7 + .../s3_scan_object/.terraform.lock.hcl | 22 ++ .../env/staging/s3_scan_object/terragrunt.hcl | 29 +++ .../staging/scan_queue/.terraform.lock.hcl | 39 +++ .../scan-queue-lock-cleanup.json | 107 ++++++++ .../scan_queue/state-machines/scan-queue.json | 244 ++++++++++++++++++ .../env/staging/scan_queue/terragrunt.hcl | 29 +++ 20 files changed, 789 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/tf_apply_staging.yml create mode 100644 .github/workflows/tf_plan_staging.yml delete mode 100644 terragrunt/env/backend.tf create mode 100644 terragrunt/env/staging/alarms/terragrunt.hcl create mode 100644 terragrunt/env/staging/api/.terraform.lock.hcl create mode 100644 terragrunt/env/staging/api/terragrunt.hcl create mode 100644 terragrunt/env/staging/env_vars.hcl create mode 100644 terragrunt/env/staging/hosted_zone/.terraform.lock.hcl create mode 100644 terragrunt/env/staging/hosted_zone/terragrunt.hcl create mode 100644 terragrunt/env/staging/integration_test/terragrunt.hcl create mode 100644 terragrunt/env/staging/s3_scan_object/.terraform.lock.hcl create mode 100644 terragrunt/env/staging/s3_scan_object/terragrunt.hcl create mode 100644 terragrunt/env/staging/scan_queue/.terraform.lock.hcl create mode 100644 terragrunt/env/staging/scan_queue/state-machines/scan-queue-lock-cleanup.json create mode 100644 terragrunt/env/staging/scan_queue/state-machines/scan-queue.json create mode 100644 terragrunt/env/staging/scan_queue/terragrunt.hcl diff --git a/.github/workflows/tf_apply_staging.yml b/.github/workflows/tf_apply_staging.yml new file mode 100644 index 00000000..a88f9314 --- /dev/null +++ b/.github/workflows/tf_apply_staging.yml @@ -0,0 +1,93 @@ +name: "Terraform apply staging" + +on: + push: + branches: + - main + paths: + - "terragrunt/**" + - ".github/workflows/tf_apply_staging.yml" + +env: + TERRAFORM_VERSION: 1.0.3 + TERRAGRUNT_VERSION: 0.38.4 + TF_VAR_api_auth_token: ${{ secrets.STAGING_API_AUTH_TOKEN }} + TF_VAR_aws_org_id: ${{ secrets.STAGING_AWS_ORG_ID }} + TF_VAR_rds_password: ${{ secrets.STAGING_RDS_PASSWORD }} + TF_VAR_slack_webhook_url: ${{ secrets.SCAN_FILES_STAGING_OPS_WEBHOOK }} + AWS_REGION: ca-central-1 + +permissions: + id-token: write + contents: read + pull-requests: write + actions: write + checks: write + statuses: write + +jobs: + terragrunt-apply: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + + - name: setup terraform tools + uses: cds-snc/terraform-tools-setup@v1 + + - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 # tag=v2.10.2 + id: filter + with: + filters: | + alarms: + - 'terragrunt/aws/alarms/**' + - 'terragrunt/env/staging/alarms/**' + api: + - 'terragrunt/aws/api/**' + - 'terragrunt/env/staging/api/**' + hosted_zone: + - 'terragrunt/aws/hosted_zone/**' + - 'terragrunt/env/staging/hosted_zone/**' + integration_test: + - 'terragrunt/aws/integration_test/**' + - 'terragrunt/env/staging/integration_test/**' + s3_scan_object: + - 'terragrunt/aws/s3_scan_object/**' + - 'terragrunt/env/staging/s3_scan_object/**' + + - name: configure aws credentials using OIDC + uses: aws-actions/configure-aws-credentials@05b148adc31e091bafbaf404f745055d4d3bc9d2 # tag=v1.6.1 + with: + role-to-assume: arn:aws:iam::127893201980:role/OIDCGithubWorkflowRole + role-session-name: TFApply + aws-region: ${{ env.AWS_REGION }} + + - name: Apply hosted_zone + if: ${{ steps.filter.outputs.hosted_zone == 'true' }} + working-directory: terragrunt/env/staging/hosted_zone + run: | + terragrunt apply --terragrunt-non-interactive -auto-approve + + - name: Apply integration_test + if: ${{ steps.filter.outputs.integration_test == 'true' }} + working-directory: terragrunt/env/staging/integration_test + run: | + terragrunt apply --terragrunt-non-interactive -auto-approve + + - name: Apply api + if: ${{ steps.filter.outputs.api == 'true' }} + working-directory: terragrunt/env/staging/api + run: | + terragrunt apply --terragrunt-non-interactive -auto-approve + + - name: Apply s3_scan_object + if: ${{ steps.filter.outputs.s3_scan_object == 'true' }} + working-directory: terragrunt/env/staging/s3_scan_object + run: | + terragrunt apply --terragrunt-non-interactive -auto-approve + + - name: Apply alarms + if: ${{ steps.filter.outputs.alarms == 'true' }} + working-directory: terragrunt/env/staging/alarms + run: | + terragrunt apply --terragrunt-non-interactive -auto-approve diff --git a/.github/workflows/tf_plan_staging.yml b/.github/workflows/tf_plan_staging.yml new file mode 100644 index 00000000..6b4c353e --- /dev/null +++ b/.github/workflows/tf_plan_staging.yml @@ -0,0 +1,60 @@ +name: "Terraform plan staging" +on: + workflow_dispatch: + pull_request: + paths: + - "terragrunt/**" + - ".github/workflows/tf_plan_staging.yml" +env: + AWS_REGION: ca-central-1 + TERRAFORM_VERSION: 1.0.3 + TERRAGRUNT_VERSION: 0.38.4 + CONFTEST_VERSION: 0.27.0 + TF_VAR_api_auth_token: ${{ secrets.STAGING_API_AUTH_TOKEN }} + TF_VAR_aws_org_id: ${{ secrets.STAGING_AWS_ORG_ID }} + TF_VAR_rds_password: ${{ secrets.STAGING_RDS_PASSWORD }} + TF_VAR_slack_webhook_url: ${{ secrets.SCAN_FILES_STAGING_OPS_WEBHOOK }} + +permissions: + id-token: write + contents: read + pull-requests: write + actions: write + checks: write + statuses: write + +jobs: + terraform-plan: + strategy: + fail-fast: false + matrix: + include: + - module: alarms + - module: api + - module: hosted_zone + - module: integration_test + - module: s3_scan_object + + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + + - name: setup terraform tools + uses: cds-snc/terraform-tools-setup@v1 + + - name: configure aws credentials using OIDC + uses: aws-actions/configure-aws-credentials@05b148adc31e091bafbaf404f745055d4d3bc9d2 # tag=v1.6.1 + with: + role-to-assume: arn:aws:iam::127893201980:role/OIDCGithubWorkflowRole + role-session-name: TFPlan + aws-region: ${{ env.AWS_REGION }} + + - name: Terraform plan + uses: cds-snc/terraform-plan@dce0ba2ac808e5c2b3499c8073adef25aaa559f3 # tag=v2.0.8 + with: + comment-delete: true + comment-title: Staging Plan for ${{ matrix.module }} + directory: ./terragrunt/env/staging/${{ matrix.module }} + github-token: ${{ secrets.GITHUB_TOKEN }} + terragrunt: true diff --git a/terragrunt/aws/api/inputs.tf b/terragrunt/aws/api/inputs.tf index 1d3b3e21..4a807ac0 100644 --- a/terragrunt/aws/api/inputs.tf +++ b/terragrunt/aws/api/inputs.tf @@ -27,3 +27,7 @@ variable "scan_queue_statemachine_name" { variable "hosted_zone_id" { type = string } + +variable "oidc_exists" { + type = bool +} diff --git a/terragrunt/aws/api/oidc_iam.tf b/terragrunt/aws/api/oidc_iam.tf index 4905c50e..2340198b 100644 --- a/terragrunt/aws/api/oidc_iam.tf +++ b/terragrunt/aws/api/oidc_iam.tf @@ -12,7 +12,7 @@ module "gh_oidc_roles" { claim = "*" } ] - oidc_exists = false + oidc_exists = var.oidc_exists billing_tag_value = var.billing_code } diff --git a/terragrunt/aws/integration_test/s3_scan_object.tf b/terragrunt/aws/integration_test/s3_scan_object.tf index cdf83e96..ce60fe21 100644 --- a/terragrunt/aws/integration_test/s3_scan_object.tf +++ b/terragrunt/aws/integration_test/s3_scan_object.tf @@ -1,8 +1,11 @@ module "integration_test" { - source = "github.com/cds-snc/terraform-modules?ref=v3.0.9//S3_scan_object" + source = "github.com/cds-snc/terraform-modules?ref=v3.0.10//S3_scan_object" - product_name = "integration-test" - s3_upload_bucket_name = module.integration_test_bucket.s3_bucket_id + product_name = "integration-test" + s3_upload_bucket_name = module.integration_test_bucket.s3_bucket_id + scan_files_role_arn = "arn:aws:iam::${var.account_id}:role/scan-files-api" + s3_scan_object_function_arn = "arn:aws:lambda:ca-central-1:${var.account_id}:function:s3-scan-object" + s3_scan_object_role_arn = "arn:aws:iam::${var.account_id}:role/s3-scan-object" billing_tag_value = var.billing_code } diff --git a/terragrunt/env/backend.tf b/terragrunt/env/backend.tf deleted file mode 100644 index 6719ce8e..00000000 --- a/terragrunt/env/backend.tf +++ /dev/null @@ -1,10 +0,0 @@ -# Generated by Terragrunt. Sig: nIlQXj57tbuaRZEa -terraform { - backend "s3" { - bucket = "scan-files-production-tf" - dynamodb_table = "terraform-state-lock-dynamo" - encrypt = true - key = "./terraform.tfstate" - region = "ca-central-1" - } -} diff --git a/terragrunt/env/production/api/terragrunt.hcl b/terragrunt/env/production/api/terragrunt.hcl index fdd9166b..d538a4fb 100644 --- a/terragrunt/env/production/api/terragrunt.hcl +++ b/terragrunt/env/production/api/terragrunt.hcl @@ -19,6 +19,7 @@ dependency "hosted_zone" { inputs = { rds_username = "databaseuser" hosted_zone_id = dependency.hosted_zone.outputs.hosted_zone_id + oidc_exists = false } include { diff --git a/terragrunt/env/staging/alarms/terragrunt.hcl b/terragrunt/env/staging/alarms/terragrunt.hcl new file mode 100644 index 00000000..ce0834de --- /dev/null +++ b/terragrunt/env/staging/alarms/terragrunt.hcl @@ -0,0 +1,25 @@ +terraform { + source = "../../../aws//alarms" +} + +dependencies { + paths = ["../api"] +} + +dependency "api" { + config_path = "../api" + + mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs = { + route53_health_check_api_id = "" + } +} + +inputs = { + route53_health_check_api_id = dependency.api.outputs.route53_health_check_api_id +} + +include { + path = find_in_parent_folders() +} diff --git a/terragrunt/env/staging/api/.terraform.lock.hcl b/terragrunt/env/staging/api/.terraform.lock.hcl new file mode 100644 index 00000000..af6cb40d --- /dev/null +++ b/terragrunt/env/staging/api/.terraform.lock.hcl @@ -0,0 +1,60 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.20.1" + constraints = ">= 3.36.0, ~> 4.9" + hashes = [ + "h1:HHfwMYY0FDtMzaGgITqsPIBlUWnQNZ5+bTF1dyscsnw=", + "zh:21d064d8fac08376c633e002e2f36e83eb7958535e251831feaf38f51c49dafd", + "zh:3a37912ff43d89ce8d559ec86265d7506801bccb380c7cfb896e8ff24e3fe79d", + "zh:795eb175c85279ec51dbe12e4d1afa0860c2c0b22e5d36a8e8869f60a93b7931", + "zh:8afb61a18b17f8ff249cb23e9d3b5d2530944001ef1d56c1d53f41b0890c7ab8", + "zh:911701040395e0e4da4b7252279e7cf1593cdd26f22835e1a9eddbdb9691a1a7", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a46d54a6a5407f569f8178e916af888b2b268f86448c64cad165dc89759c8399", + "zh:c5f71fd5e3519a24fd6af455ef1c26a559cfdde7f626b0afbd2a73bb79f036b1", + "zh:df3b69d6c9b0cdc7e3f90ee08412b22332c32e97ad8ce6ccad528f89f235a7d3", + "zh:e99d6a64c03549d60c2accf792fa04466cfb317f72e895c8f67eff8a02920887", + "zh:eea7a0df8bcb69925c9ce8e15ef403c8bbf16d46c43e8f5607b116531d1bce4a", + "zh:f6a26ce77f7db1d50ce311e32902fd001fb365e5e45e47a9a5cd59d734c89cb6", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.3.2" + hashes = [ + "h1:H5V+7iXol/EHB2+BUMzGlpIiCOdV74H8YjzCxnSAWcg=", + "zh:038293aebfede983e45ee55c328e3fde82ae2e5719c9bd233c324cfacc437f9c", + "zh:07eaeab03a723d83ac1cc218f3a59fceb7bbf301b38e89a26807d1c93c81cef8", + "zh:427611a4ce9d856b1c73bea986d841a969e4c2799c8ac7c18798d0cc42b78d32", + "zh:49718d2da653c06a70ba81fd055e2b99dfd52dcb86820a6aeea620df22cd3b30", + "zh:5574828d90b19ab762604c6306337e6cd430e65868e13ef6ddb4e25ddb9ad4c0", + "zh:7222e16f7833199dabf1bc5401c56d708ec052b2a5870988bc89ff85b68a5388", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:b1b2d7d934784d2aee98b0f8f07a8ccfc0410de63493ae2bf2222c165becf938", + "zh:b8f85b6a20bd264fcd0814866f415f0a368d1123cd7879c8ebbf905d370babc8", + "zh:c3813133acc02bbebddf046d9942e8ba5c35fc99191e3eb057957dafc2929912", + "zh:e7a41dbc919d1de800689a81c240c27eec6b9395564630764ebb323ea82ac8a9", + "zh:ee6d23208449a8eaa6c4f203e33f5176fa795b4b9ecf32903dffe6e2574732c2", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "3.4.0" + hashes = [ + "h1:oyllIA9rNGCFtClSyBitUIzCXdnKtspVepdsvpLlfys=", + "zh:2442a0df0cfb550b8eba9b2af39ac06f54b62447eb369ecc6b1c29f739b33bbb", + "zh:3ebb82cacb677a099de55f844f0d02886bc804b1a2b94441bc40fabcb64d2a38", + "zh:436125c2a7e66bc62a4a7c68bdca694f071d7aa894e8637dc83f4a68fe322546", + "zh:5f03db9f1d77e8274ff4750ae32d5c16c42b862b06bcb0683e4d733c8db922e4", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8190142ae8a539ab34193b7e75da0fa04035d1dcd8af8be94df1eafeeffb44b6", + "zh:8cdc7cd9221e27c189e5beaf78462fce4c2edb081f415a1eafc6da2949de31e2", + "zh:a5de0f7f5d63c59ebf61d3c1d94040f410665ff0aa04f66674efe24b39a11f94", + "zh:a9fce48db3c140cc3e06f8a3c7ef4d36735e457e7660442d6d5dcd2b0781adc3", + "zh:beb92de584c790c7c7f047e45ccd22b6ee3263c7b5a91ae4d6882ae6e7700570", + "zh:f373f8cc52846fb513f44f468d885f722ca4dc22af9ff1942368cafd16b796b3", + "zh:f69627fd6e5a920b17ff423cdbad2715078ca6d13146dc67668795582ab43748", + ] +} diff --git a/terragrunt/env/staging/api/terragrunt.hcl b/terragrunt/env/staging/api/terragrunt.hcl new file mode 100644 index 00000000..2586a3fd --- /dev/null +++ b/terragrunt/env/staging/api/terragrunt.hcl @@ -0,0 +1,27 @@ +terraform { + source = "../../../aws//api" +} + +dependencies { + paths = ["../hosted_zone"] +} + +dependency "hosted_zone" { + config_path = "../hosted_zone" + + mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs = { + hosted_zone_id = "" + } +} + +inputs = { + rds_username = "databaseuser" + hosted_zone_id = dependency.hosted_zone.outputs.hosted_zone_id + oidc_exists = true +} + +include { + path = find_in_parent_folders() +} \ No newline at end of file diff --git a/terragrunt/env/staging/env_vars.hcl b/terragrunt/env/staging/env_vars.hcl new file mode 100644 index 00000000..86646507 --- /dev/null +++ b/terragrunt/env/staging/env_vars.hcl @@ -0,0 +1,6 @@ +inputs = { + account_id = "127893201980" + env = "staging" + cost_center_code = "scan-files-staging" + domain = "scan-files.cdssandbox.xyz" +} \ No newline at end of file diff --git a/terragrunt/env/staging/hosted_zone/.terraform.lock.hcl b/terragrunt/env/staging/hosted_zone/.terraform.lock.hcl new file mode 100644 index 00000000..94683924 --- /dev/null +++ b/terragrunt/env/staging/hosted_zone/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.20.1" + constraints = "~> 4.9" + hashes = [ + "h1:HHfwMYY0FDtMzaGgITqsPIBlUWnQNZ5+bTF1dyscsnw=", + "zh:21d064d8fac08376c633e002e2f36e83eb7958535e251831feaf38f51c49dafd", + "zh:3a37912ff43d89ce8d559ec86265d7506801bccb380c7cfb896e8ff24e3fe79d", + "zh:795eb175c85279ec51dbe12e4d1afa0860c2c0b22e5d36a8e8869f60a93b7931", + "zh:8afb61a18b17f8ff249cb23e9d3b5d2530944001ef1d56c1d53f41b0890c7ab8", + "zh:911701040395e0e4da4b7252279e7cf1593cdd26f22835e1a9eddbdb9691a1a7", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a46d54a6a5407f569f8178e916af888b2b268f86448c64cad165dc89759c8399", + "zh:c5f71fd5e3519a24fd6af455ef1c26a559cfdde7f626b0afbd2a73bb79f036b1", + "zh:df3b69d6c9b0cdc7e3f90ee08412b22332c32e97ad8ce6ccad528f89f235a7d3", + "zh:e99d6a64c03549d60c2accf792fa04466cfb317f72e895c8f67eff8a02920887", + "zh:eea7a0df8bcb69925c9ce8e15ef403c8bbf16d46c43e8f5607b116531d1bce4a", + "zh:f6a26ce77f7db1d50ce311e32902fd001fb365e5e45e47a9a5cd59d734c89cb6", + ] +} diff --git a/terragrunt/env/staging/hosted_zone/terragrunt.hcl b/terragrunt/env/staging/hosted_zone/terragrunt.hcl new file mode 100644 index 00000000..97260336 --- /dev/null +++ b/terragrunt/env/staging/hosted_zone/terragrunt.hcl @@ -0,0 +1,7 @@ +terraform { + source = "../../../aws//hosted_zone" +} + +include { + path = find_in_parent_folders() +} diff --git a/terragrunt/env/staging/integration_test/terragrunt.hcl b/terragrunt/env/staging/integration_test/terragrunt.hcl new file mode 100644 index 00000000..766d7a8e --- /dev/null +++ b/terragrunt/env/staging/integration_test/terragrunt.hcl @@ -0,0 +1,7 @@ +terraform { + source = "../../../aws//integration_test" +} + +include { + path = find_in_parent_folders() +} diff --git a/terragrunt/env/staging/s3_scan_object/.terraform.lock.hcl b/terragrunt/env/staging/s3_scan_object/.terraform.lock.hcl new file mode 100644 index 00000000..94683924 --- /dev/null +++ b/terragrunt/env/staging/s3_scan_object/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.20.1" + constraints = "~> 4.9" + hashes = [ + "h1:HHfwMYY0FDtMzaGgITqsPIBlUWnQNZ5+bTF1dyscsnw=", + "zh:21d064d8fac08376c633e002e2f36e83eb7958535e251831feaf38f51c49dafd", + "zh:3a37912ff43d89ce8d559ec86265d7506801bccb380c7cfb896e8ff24e3fe79d", + "zh:795eb175c85279ec51dbe12e4d1afa0860c2c0b22e5d36a8e8869f60a93b7931", + "zh:8afb61a18b17f8ff249cb23e9d3b5d2530944001ef1d56c1d53f41b0890c7ab8", + "zh:911701040395e0e4da4b7252279e7cf1593cdd26f22835e1a9eddbdb9691a1a7", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a46d54a6a5407f569f8178e916af888b2b268f86448c64cad165dc89759c8399", + "zh:c5f71fd5e3519a24fd6af455ef1c26a559cfdde7f626b0afbd2a73bb79f036b1", + "zh:df3b69d6c9b0cdc7e3f90ee08412b22332c32e97ad8ce6ccad528f89f235a7d3", + "zh:e99d6a64c03549d60c2accf792fa04466cfb317f72e895c8f67eff8a02920887", + "zh:eea7a0df8bcb69925c9ce8e15ef403c8bbf16d46c43e8f5607b116531d1bce4a", + "zh:f6a26ce77f7db1d50ce311e32902fd001fb365e5e45e47a9a5cd59d734c89cb6", + ] +} diff --git a/terragrunt/env/staging/s3_scan_object/terragrunt.hcl b/terragrunt/env/staging/s3_scan_object/terragrunt.hcl new file mode 100644 index 00000000..0b9ea6e2 --- /dev/null +++ b/terragrunt/env/staging/s3_scan_object/terragrunt.hcl @@ -0,0 +1,29 @@ +terraform { + source = "../../../aws//s3_scan_object" +} + +dependencies { + paths = ["../api"] +} + +dependency "api" { + config_path = "../api" + + mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"] + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs = { + function_name = "" + function_role_arn = "" + scan_files_api_key_secret_arn = "" + } +} + +inputs = { + scan_files_api_function_role_arn = dependency.api.outputs.function_role_arn + scan_files_api_function_role_name = dependency.api.outputs.function_name + scan_files_api_key_secret_arn = dependency.api.outputs.scan_files_api_key_secret_arn +} + +include { + path = find_in_parent_folders() +} diff --git a/terragrunt/env/staging/scan_queue/.terraform.lock.hcl b/terragrunt/env/staging/scan_queue/.terraform.lock.hcl new file mode 100644 index 00000000..71cd518d --- /dev/null +++ b/terragrunt/env/staging/scan_queue/.terraform.lock.hcl @@ -0,0 +1,39 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.20.1" + constraints = "~> 4.9" + hashes = [ + "h1:HHfwMYY0FDtMzaGgITqsPIBlUWnQNZ5+bTF1dyscsnw=", + "zh:21d064d8fac08376c633e002e2f36e83eb7958535e251831feaf38f51c49dafd", + "zh:3a37912ff43d89ce8d559ec86265d7506801bccb380c7cfb896e8ff24e3fe79d", + "zh:795eb175c85279ec51dbe12e4d1afa0860c2c0b22e5d36a8e8869f60a93b7931", + "zh:8afb61a18b17f8ff249cb23e9d3b5d2530944001ef1d56c1d53f41b0890c7ab8", + "zh:911701040395e0e4da4b7252279e7cf1593cdd26f22835e1a9eddbdb9691a1a7", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:a46d54a6a5407f569f8178e916af888b2b268f86448c64cad165dc89759c8399", + "zh:c5f71fd5e3519a24fd6af455ef1c26a559cfdde7f626b0afbd2a73bb79f036b1", + "zh:df3b69d6c9b0cdc7e3f90ee08412b22332c32e97ad8ce6ccad528f89f235a7d3", + "zh:e99d6a64c03549d60c2accf792fa04466cfb317f72e895c8f67eff8a02920887", + "zh:eea7a0df8bcb69925c9ce8e15ef403c8bbf16d46c43e8f5607b116531d1bce4a", + "zh:f6a26ce77f7db1d50ce311e32902fd001fb365e5e45e47a9a5cd59d734c89cb6", + ] +} + +provider "registry.terraform.io/hashicorp/template" { + version = "2.2.0" + hashes = [ + "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", + "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", + "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", + "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", + "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", + "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", + "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", + "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", + "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", + "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", + "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", + ] +} diff --git a/terragrunt/env/staging/scan_queue/state-machines/scan-queue-lock-cleanup.json b/terragrunt/env/staging/scan_queue/state-machines/scan-queue-lock-cleanup.json new file mode 100644 index 00000000..4a0d33f9 --- /dev/null +++ b/terragrunt/env/staging/scan_queue/state-machines/scan-queue-lock-cleanup.json @@ -0,0 +1,107 @@ +{ + "Comment": "A state machine that will react to completion events and clean up orphaned locks", + "StartAt": "Get Current Lock Item", + "States": { + "Get Current Lock Item": { + "Comment": "Get info from DDB for the lock item to look and see if this specific owner is still holding a lock", + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${table_semaphore}", + "ExpressionAttributeNames": { + "#lockownerid.$": "$.detail.executionArn" + }, + "Key": { + "LockName": { + "S": "${lock_name}" + } + }, + "ProjectionExpression": "#lockownerid" + }, + "Retry": [ + { + "ErrorEquals": [ "States.ALL" ], + "IntervalSeconds": 5, + "MaxAttempts": 20, + "BackoffRate": 1.4 + } ], + "ResultSelector": { + "Item.$": "$.Item", + "ItemString.$": "States.JsonToString($.Item)" + }, + "ResultPath": "$.lockinfo.currentlockitem", + "Next": "Check If Lock Is Held" + }, + "Check If Lock Is Held": { + "Comment": "This state checks to see if the execution in question holds a lock. It can tell that by looking for Z, which will be indicative of the timestamp value. That will only be there in the stringified version of the data returned from DDB if this execution holds a lock", + "Type": "Choice", + "Choices": [ + { + "And": [ + { + "Variable": "$.lockinfo.currentlockitem.ItemString", + "IsPresent": true + }, + { + "Variable": "$.lockinfo.currentlockitem.ItemString", + "StringMatches": "*Z*" + } + ], + "Next": "Clean Up Lock" + } + ], + "Default": "Success State" + }, + "Clean Up Lock": { + "Comment": "If this lockownerid is still there, then clean it up and release the lock", + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:updateItem", + "Parameters": { + "TableName": "${table_semaphore}", + "Key": { + "LockName": { + "S": "${lock_name}" + } + }, + "ExpressionAttributeNames": { + "#currentlockcount": "currentlockcount", + "#lockownerid.$": "$.detail.executionArn" + }, + "ExpressionAttributeValues": { + ":decrease": { + "N": "1" + } + }, + "UpdateExpression": "SET #currentlockcount = #currentlockcount - :decrease REMOVE #lockownerid", + "ConditionExpression": "attribute_exists(#lockownerid)", + "ReturnValues": "UPDATED_NEW" + }, + "Retry": [ + { + "ErrorEquals": [ "DynamoDB.ConditionalCheckFailedException" ], + "MaxAttempts": 0 + }, + { + "ErrorEquals": [ "States.ALL" ], + "IntervalSeconds": 5, + "MaxAttempts": 20, + "BackoffRate": 1.4 + } ], + "Catch": [ + { + "ErrorEquals": [ + "DynamoDB.ConditionalCheckFailedException" + ], + "Next": "Success State", + "ResultPath": null + } + ], + "ResultPath": null, + "Next": "Success State" + }, + "Success State": { + "Type": "Succeed" + } + } + +} \ No newline at end of file diff --git a/terragrunt/env/staging/scan_queue/state-machines/scan-queue.json b/terragrunt/env/staging/scan_queue/state-machines/scan-queue.json new file mode 100644 index 00000000..774879f4 --- /dev/null +++ b/terragrunt/env/staging/scan_queue/state-machines/scan-queue.json @@ -0,0 +1,244 @@ +{ + "StartAt": "Get Lock", + "TimeoutSeconds": 14400, + "States": { + "Get Lock": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Acquire Lock", + "States": { + "Acquire Lock": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:updateItem", + "Parameters": { + "TableName": "${table_semaphore}", + "Key": { + "LockName": { + "S": "${lock_name}" + } + }, + "ExpressionAttributeNames": { + "#currentlockcount": "currentlockcount", + "#lockownerid.$": "$$.Execution.Id" + }, + "ExpressionAttributeValues": { + ":increase": { + "N": "1" + }, + ":limit": { + "N": "${concurrent_scan_limit}" + }, + ":lockacquiredtime": { + "S.$": "$$.State.EnteredTime" + } + }, + "UpdateExpression": "SET #currentlockcount = #currentlockcount + :increase, #lockownerid = :lockacquiredtime", + "ConditionExpression": "currentlockcount <> :limit and attribute_not_exists(#lockownerid)", + "ReturnValues": "UPDATED_NEW" + }, + "Retry": [ + { + "ErrorEquals": ["DynamoDB.AmazonDynamoDBException"], + "MaxAttempts": 0 + }, + { + "ErrorEquals": [ "States.ALL" ], + "MaxAttempts": 6, + "BackoffRate": 2 + } + ], + "Catch": [ + { + "ErrorEquals": ["DynamoDB.AmazonDynamoDBException"], + "Next": "Initialize Lock Item", + "ResultPath": "$.lockinfo.acquisitionerror" + }, + { + "ErrorEquals": [ + "DynamoDB.ConditionalCheckFailedException" + ], + "Next": "Get Current Lock Record", + "ResultPath": "$.lockinfo.acquisitionerror" + } + ], + "End": true + }, + "Initialize Lock Item": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:putItem", + "Parameters": { + "TableName": "${table_semaphore}", + "Item": { + "LockName": { + "S": "${lock_name}" + }, + "currentlockcount": { + "N": "0" + } + }, + "ConditionExpression": "LockName <> :lockname", + "ExpressionAttributeValues": { + ":lockname": {"S": "${lock_name}"} + } + }, + "Catch": [ + { + "ErrorEquals": [ "States.ALL" ], + "Next": "Acquire Lock", + "ResultPath": null + } + ], + "Next": "Acquire Lock", + "ResultPath": null + }, + "Get Current Lock Record": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "${table_semaphore}", + "ExpressionAttributeNames": { + "#lockownerid.$": "$$.Execution.Id" + }, + "Key": { + "LockName": { + "S": "${lock_name}" + } + }, + "ProjectionExpression": "#lockownerid" + }, + "ResultSelector": { + "Item.$": "$.Item", + "ItemString.$": "States.JsonToString($.Item)" + }, + "ResultPath": "$.lockinfo.currentlockitem", + "Next": "Check If Lock Already Acquired" + }, + "Check If Lock Already Acquired": { + "Type": "Choice", + "Choices": [ + { + "And": [ + { + "Variable": "$.lockinfo.currentlockitem.ItemString", + "IsPresent": true + }, + { + "Variable": "$.lockinfo.currentlockitem.ItemString", + "StringMatches": "*Z*" + } + ], + "Next": "Continue Because Lock Was Already Acquired" + } + ], + "Default": "Wait to Get Lock" + }, + "Continue Because Lock Was Already Acquired": { + "Type": "Pass", + "End": true + }, + "Wait to Get Lock": { + "Type": "Wait", + "Seconds": ${retry_interval_seconds}, + "Next": "Acquire Lock" + } + } + } + ], + "ResultPath": null, + "Next": "Start AssemblyLine scan" + }, + "Start AssemblyLine scan": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${worker_lambda}", + "Payload": { + "task": "assemblyline_scan", + "Input.$": "$", + "execution_id.$": "$$.Execution.Id" + } + }, + "ResultPath": "$.payload" , + "Next": "Get AssemblyLine Result" + }, + "Get AssemblyLine Result": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${worker_lambda}", + "Payload": { + "task": "assemblyline_result", + "Input.$": "$", + "execution_id.$": "$$.Execution.Id" + } + }, + "ResultPath": "$.payload", + "Next": "Check If Results Available" + }, + "Check If Results Available": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.payload.Payload", + "BooleanEquals": true, + "Next": "Release Lock" + } + ], + "Default": "Wait to Get Results" + }, + "Wait to Get Results": { + "Type": "Wait", + "Seconds": ${retry_interval_seconds}, + "Next": "Get AssemblyLine Result" + }, + "Release Lock": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:updateItem", + "Parameters": { + "TableName": "${table_semaphore}", + "Key": { + "LockName": { + "S": "${lock_name}" + } + }, + "ExpressionAttributeNames": { + "#currentlockcount": "currentlockcount", + "#lockownerid.$": "$$.Execution.Id" + }, + "ExpressionAttributeValues": { + ":decrease": { + "N": "1" + } + }, + "UpdateExpression": "SET #currentlockcount = #currentlockcount - :decrease REMOVE #lockownerid", + "ConditionExpression": "attribute_exists(#lockownerid)", + "ReturnValues": "UPDATED_NEW" + }, + "Retry": [ + { + "ErrorEquals": [ "DynamoDB.ConditionalCheckFailedException" ], + "MaxAttempts": 0 + }, + { + "ErrorEquals": [ "States.ALL" ], + "MaxAttempts": 5, + "BackoffRate": 1.5 + } ], + "Catch": [ + { + "ErrorEquals": [ + "DynamoDB.ConditionalCheckFailedException" + ], + "Next": "Success State", + "ResultPath": null + } + ], + "ResultPath": null, + "Next": "Success State" + }, + "Success State": { + "Type": "Succeed" + } + } + } \ No newline at end of file diff --git a/terragrunt/env/staging/scan_queue/terragrunt.hcl b/terragrunt/env/staging/scan_queue/terragrunt.hcl new file mode 100644 index 00000000..76125ecc --- /dev/null +++ b/terragrunt/env/staging/scan_queue/terragrunt.hcl @@ -0,0 +1,29 @@ +terraform { + source = "../../../aws//scan_queue" +} + +dependencies { + paths = ["../api"] +} + +dependency "api" { + config_path = "../api" + + mock_outputs_allowed_terraform_commands = ["validate", "plan"] + mock_outputs_merge_strategy_with_state = "shallow" + mock_outputs = { + function_arn = "" + function_name = "" + } +} + +inputs = { + concurrent_scan_limit = 5 + retry_interval_seconds = 120 + api_function_arn = dependency.api.outputs.function_arn + api_function_name = dependency.api.outputs.function_name +} + +include { + path = find_in_parent_folders() +} \ No newline at end of file