Add CDK and workflow for provisioning OpenSearch service (#700)

* Add CDK and workflow for provisioning OpenSearch service
* Update some Lambda Python runtime from 3.8 to 3.12

.github/workflows/opensearch-build.yml

@@ -0,0 +1,42 @@
+name: OpenSearch - Build
+run-name: Test IaC @ ${{ github.ref_name }}
+
+on:
+  push:
+    paths:
+      - .github/workflows/opensearch-build.yml
+      - OpenSearch/cdk/**
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  bedrock-agent-aoss:
+    name: Test OpenSearch Service IaC
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: OpenSearch/cdk/opensearchservice
+    env:
+      ENV_STAGE: dev
+    steps:
+      - uses: actions/checkout@v4
+      - run: make lint-python
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+      - name: Set up aws-cdk
+        run: make install-cdk
+      - name: Print deployment environment
+        run: |
+          echo "INFO: cdk version: $(cdk --version)"
+          echo "INFO: node version: $(node --version)"
+          echo "INFO: npm version: $(npm --version)"
+          echo "INFO: python3 version: $(python3 --version)"
+      - name: Run cdk synth
+        run: make synth

APIGateway/API-Canary-SFN-Lambda/cfn/Apigw-Sfn-Lambda.template.yaml

@@ -79,7 +79,7 @@ Resources:
       Handler: index.lambda_handler
       MemorySize: 1024
       Role: !GetAtt LambdaExecutionRole.Arn
-      Runtime: python3.8
+      Runtime: python3.12
       Timeout: 900
       TracingConfig:
         Mode: !Ref FunctionTracingConfigMode
@@ -156,7 +156,7 @@ Resources:
       Handler: index.lambda_handler
       MemorySize: 1024
       Role: !GetAtt LambdaExecutionRole.Arn
-      Runtime: python3.8
+      Runtime: python3.12
       Timeout: 900
       TracingConfig:
         Mode: !Ref FunctionTracingConfigMode

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 All notable changes to this project will be documented in this file.
 
+## 2024-10-17
+
+### Added
+   * Added [OpenSearch/cdk/opensearchservice/](OpenSearch/cdk/opensearchservice/) - CDK and workflow for creating OpenSearch Service.
 
 ## 2024-10-11
 

Lambda/cfn/Lambda-Init.template.yaml

@@ -41,7 +41,7 @@ Resources:
       Handler: index.lambda_handler
       MemorySize: 1024
       Role: !GetAtt LambdaExecutionRole.Arn
-      Runtime: python3.8
+      Runtime: python3.12
       Timeout: 30
       TracingConfig:
         Mode: Active

OpenSearch/cdk/opensearchservice/Makefile

@@ -0,0 +1,44 @@
+export AWS_DEFAULT_REGION ?= ap-southeast-2
+export CDK_DEFAULT_REGION ?= ap-southeast-2
+export ENV_STAGE ?= dev
+
+APP_NAME=$(shell grep -m 1 AppName environment/$(ENV_STAGE).yml | cut -c 10-)
+
+install-cdk:
+	npm install -g aws-cdk
+	python3 -m pip install -U pip
+	pip3 install -r requirements.txt
+
+synth:
+	cdk synth $(APP_NAME)-OpenSearch -c env=$(ENV_STAGE)
+
+diff:
+	cdk diff $(APP_NAME)-OpenSearch -c env=$(ENV_STAGE)
+
+deploy:
+	cdk deploy $(APP_NAME)-OpenSearch -c env=$(ENV_STAGE) $(APP_NAME) --require-approval never
+
+destroy:
+	cdk destroy $(APP_NAME)-OpenSearch -f -c env=$(ENV_STAGE)
+
+test-cdk:
+	pip3 install -r requirements-dev.txt && \
+	python3 -m pytest .
+
+test-code:
+	python3 tests/test_guardrail.py
+
+pre-commit: format-python lint-python lint-yaml
+
+format-python:
+	black **.py */**.py
+
+lint-python:
+	pip3 install flake8
+	flake8 **.py */**.py
+
+lint-yaml:
+	yamllint -c .github/linters/.yaml-lint.yml -f parsable .
+
+clean:
+	rm -rf cdk.out __pycache__

OpenSearch/cdk/opensearchservice/app.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+from os.path import dirname, join, realpath
+
+import yaml
+from aws_cdk import App, Environment, Tags
+from lib.opensearch import OpenSearchServiceStack
+
+ENV_DIR = join(dirname(realpath(__file__)), "environment")
+
+
+def main():
+    app = App()
+
+    ENV_NAME = app.node.try_get_context("env") or "dev"
+
+    with open(join(ENV_DIR, f"{ENV_NAME}.yml"), "r") as stream:
+        yaml_data = yaml.safe_load(stream)
+        config = yaml_data if yaml_data is not None else {}
+
+    app_name = config["AppName"]
+
+    stack = OpenSearchServiceStack(
+        scope=app,
+        id=f"{app_name}-OpenSearch",
+        config=config,
+        env=Environment(account=config["Account"], region=config["Region"]),
+    )
+
+    for key, value in config["Tags"].items():
+        Tags.of(stack).add(key, value)
+
+    app.synth()
+
+
+if __name__ == "__main__":
+    main()

OpenSearch/cdk/opensearchservice/cdk.json

@@ -0,0 +1,26 @@
+{
+    "app": "python3 app.py",
+    "watch": {
+        "include": ["**"],
+        "exclude": [
+            "README.md",
+            "cdk*.json",
+            "requirements*.txt",
+            "**/__init__.py",
+            "**/__pycache__",
+            "tests"
+        ]
+    },
+    "context": {
+        "@aws-cdk/core:checkSecretUsage": true,
+        "@aws-cdk/core:enablePartitionLiterals": true,
+        "@aws-cdk/core:includePrefixInUniqueNameGeneration": true,
+        "@aws-cdk/core:target-partitions": ["aws", "aws-cn"],
+        "@aws-cdk/aws-iam:minimizePolicies": true,
+        "@aws-cdk/aws-iam:standardizedServicePrincipals": true,
+        "@aws-cdk/aws-kms:aliasNameRef": true,
+        "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true,
+        "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true,
+        "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true
+    }
+}

OpenSearch/cdk/opensearchservice/environment/dev.yml

@@ -0,0 +1,26 @@
+AppName: Test
+Account: "123456789012"
+Region: ap-southeast-2
+Kms:
+  key_admin_arns:
+    - arn:aws:iam::123456789012:role/key-admin
+    - arn:aws:iam::123456789012:role/deploy-role
+  key_user_arns:
+    - arn:aws:iam::123456789012:role/developer-*
+OpenSearch:
+  domain_admin_uname: opensearch
+  domain_data_node_instance_type: m6g.large.search
+  domain_data_node_instance_count: 3
+  domain_instance_volume_size: 100
+  domain_az_count: 3
+  ## Maximum Master Instance count supported by service is 5, so either have 3 or 5 dedicated node for master
+  domain_master_node_instance_type: m6g.large.search
+  domain_master_node_instance_count: 3
+  ## To enable UW, please make master node count as 3 or 5, and UW node count as minimum 2
+  ## Also change data node to be non T2/T3 as UW does not support T2/T3 as data nodes
+  domain_uw_node_instance_type: ultrawarm1.medium.search
+  domain_uw_node_instance_count: 0
+
+Tags:
+  CostCentre: TODO
+  Project: TODO

OpenSearch/cdk/opensearchservice/lib/base_infra.py

@@ -0,0 +1,79 @@
+import os
+
+from aws_cdk import aws_ec2 as ec2
+from aws_cdk.aws_ssm import StringParameter
+from constructs import Construct
+
+CDK_LOCAL_SYNC = os.environ.get("CDK_LOCAL_SYNC", "true").lower() == "true"  # with no credentials
+
+AZS = ["ap-southeast-2"]
+SUBNET_AZs = [
+    "ap-southeast-2a",
+    "ap-southeast-2b",
+    "ap-southeast-2c",
+]
+SSM_PARAM_VPC01_ID = "/account/vpc01-id"
+SSM_PARAM_VPC01_SUBNET_APP_IDs = [
+    "/account/vpc01-subnet-app-a-id",
+    "/account/vpc01-subnet-app-b-id",
+    "/account/vpc01-subnet-app-c-id",
+]
+SSM_PARAM_VPC01_APP_CIDR = "/account/vpc01-app-cidr"
+SSM_PARAM_VPC01_SG_INT_USERS_ID = "/account/vpc01-securitygroup-int-users-id"
+SSM_PARAM_VPC01_ROUTE_TABLE_IDs = [
+    "/account/vpc01-routetable-app-a-id",
+    "/account/vpc01-routetable-app-b-id",
+    "/account/vpc01-routetable-app-c-id",
+]
+
+
+class BaseInfra(Construct):
+    def __init__(self, scope: Construct, app_name: str) -> None:
+        super().__init__(scope, "BaseInfra")
+
+        self.app_name = app_name
+        self.base_stack_name = app_name.lower()
+
+        self.app_subnet_ids = [
+            self._value_from_lookup(ssm_param) for ssm_param in SSM_PARAM_VPC01_SUBNET_APP_IDs
+        ]
+        self.app_subnets = self.get_app_subnets(scope)
+
+        self.vpc_id = self._value_from_lookup(SSM_PARAM_VPC01_ID)
+        self.app_vpc = ec2.Vpc.from_vpc_attributes(
+            self,
+            "AppVpc",
+            vpc_id=self.vpc_id,
+            availability_zones=AZS,
+            private_subnet_ids=self.app_subnet_ids,
+        )
+
+        self.app_subnet_cidr = self._value_from_lookup(
+            SSM_PARAM_VPC01_APP_CIDR, mock_value="10.0.0.0/28"
+        )
+
+        self.int_users_sg = ec2.SecurityGroup.from_security_group_id(
+            self,
+            "InternalUsersSG",
+            self._value_from_lookup(SSM_PARAM_VPC01_SG_INT_USERS_ID),
+        )
+
+    def get_app_subnets(self, scope: Construct, prefix: str = "default") -> list:
+        app_subnets = []
+        for i, subnet_id in enumerate(self.app_subnet_ids):
+            route_table_id = self._value_from_lookup(SSM_PARAM_VPC01_ROUTE_TABLE_IDs[i])
+            app_subnets.append(
+                ec2.Subnet.from_subnet_attributes(
+                    scope,
+                    f"{prefix}AppSubnet{i}",
+                    subnet_id=subnet_id,
+                    availability_zone=SUBNET_AZs[i],
+                    route_table_id=route_table_id,
+                )
+            )
+        return app_subnets
+
+    def _value_from_lookup(self, param_name: str, mock_value=None) -> str:
+        if CDK_LOCAL_SYNC is True:
+            return mock_value if mock_value else f'mock-{param_name.replace("/", "-")}'
+        return StringParameter.value_from_lookup(self, param_name)

OpenSearch/cdk/opensearchservice/lib/kms.py

@@ -0,0 +1,58 @@
+from aws_cdk import aws_iam as iam
+from aws_cdk import aws_kms as kms
+from constructs import Construct
+
+
+def create_kms_key_and_alias(
+    scope: Construct,
+    id: str,
+    key_alias: str,
+    key_admin_arns: list,
+    key_user_arns_like: list,
+    enable_key_rotation: bool = False,
+) -> kms.Key:
+    policy_document = iam.PolicyDocument(
+        statements=[
+            iam.PolicyStatement(
+                actions=["kms:*"],
+                principals=[iam.AccountRootPrincipal()],
+                resources=["*"],
+                sid="EnableIAMUserPermissions",
+            ),
+            iam.PolicyStatement(
+                actions=[
+                    "kms:Create*",
+                    "kms:Describe*",
+                    "kms:Enable*",
+                    "kms:List*",
+                    "kms:Put*",
+                    "kms:Update*",
+                    "kms:Revoke*",
+                    "kms:Disable*",
+                    "kms:Get*",
+                    "kms:Delete*",
+                    "kms:TagResource",
+                    "kms:UntagResource",
+                    "kms:ScheduleKeyDeletion",
+                    "kms:CancelKeyDeletion",
+                ],
+                principals=[
+                    iam.Role.from_role_arn(scope, arn_str.split("/")[-1], arn_str)
+                    for arn_str in key_admin_arns
+                ],
+                resources=["*"],
+                sid="AllowAccessForKeyAdministrators",
+            ),
+            iam.PolicyStatement(
+                actions=["kms:Decrypt"],
+                conditions={"ArnLike": {"aws:PrincipalArn": key_user_arns_like}},
+                principals=[iam.AnyPrincipal()],
+                resources=["*"],
+                sid="AllowDecrypt",
+            ),
+        ]
+    )
+
+    key = kms.Key(scope, id, enable_key_rotation=enable_key_rotation, policy=policy_document)
+
+    return kms.Alias(scope, f"{id}Alias", alias_name=key_alias, target_key=key)