Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactors SCHISM FIM workflow to use AWS Batch w/EC2 compute environment rather than Lambda to save costs #722

Merged
merged 5 commits into from
May 16, 2024
Merged

Refactors SCHISM FIM workflow to use AWS Batch w/EC2 compute environment rather than Lambda to save costs #722

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
02c9979
Fully implemented
shawncrawley May 6, 2024
2962e7f
Passes along a few variables that had been missed
shawncrawley May 7, 2024
2602f96
Replaces hard-coded account ID with variable
shawncrawley May 7, 2024
8bcdd2c
Fix relative paths and explicitly list file to zip
shawncrawley May 7, 2024
ebc2558
fixes made to allow deployment
nickchadwick-noaa May 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 22 additions & 140 deletions Core/LAMBDA/viz_functions/image_based/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -484,152 +484,34 @@ data "aws_lambda_function" "viz_hand_fim_processing" {
}


##################################
## SCHISM HUC PROCESSING LAMBDA ##
##################################

data "archive_file" "schism_processing_zip" {
type = "zip"
output_path = "${path.module}/temp/viz_schism_fim_processing_${var.environment}_${var.region}.zip"

dynamic "source" {
for_each = fileset("${path.module}/viz_schism_fim_processing", "**")
content {
content = file("${path.module}/viz_schism_fim_processing/${source.key}")
filename = source.key
}
}

source {
content = file("${path.module}/../../layers/viz_lambda_shared_funcs/python/viz_classes.py")
filename = "viz_classes.py"
}

source {
content = templatefile("${path.module}/viz_schism_fim_processing/serverless.yml.tmpl", {
SERVICE_NAME = replace(local.viz_schism_fim_processing_lambda_name, "_", "-")
LAMBDA_TAGS = jsonencode(merge(var.default_tags, { Name = local.viz_schism_fim_processing_lambda_name }))
DEPLOYMENT_BUCKET = var.deployment_bucket
AWS_DEFAULT_REGION = var.region
LAMBDA_NAME = local.viz_schism_fim_processing_lambda_name
AWS_ACCOUNT_ID = var.account_id
IMAGE_REPO_NAME = aws_ecr_repository.viz_schism_fim_processing_image.name
IMAGE_TAG = var.ecr_repository_image_tag
LAMBDA_ROLE_ARN = var.lambda_role
PYTHON_PREPROCESSING_BUCKET = var.python_preprocessing_bucket
INPUTS_BUCKET = var.deployment_bucket
INPUTS_PREFIX = "schism_fim"
VIZ_DB_DATABASE = var.viz_db_name
VIZ_DB_HOST = var.viz_db_host
VIZ_DB_PASSWORD = jsondecode(var.viz_db_user_secret_string)["password"]
VIZ_DB_USERNAME = jsondecode(var.viz_db_user_secret_string)["username"]
SECURITY_GROUP_1 = var.hand_fim_processing_sgs[0]
SUBNET_1 = var.hand_fim_processing_subnets[0]
SUBNET_2 = var.hand_fim_processing_subnets[1]
})
filename = "serverless.yml"
}
}

resource "aws_s3_object" "schism_processing_zip_upload" {
bucket = var.deployment_bucket
key = "terraform_artifacts/${path.module}/viz_schism_fim_processing.zip"
source = data.archive_file.schism_processing_zip.output_path
source_hash = data.archive_file.schism_processing_zip.output_md5
}

resource "aws_ecr_repository" "viz_schism_fim_processing_image" {
name = local.viz_schism_fim_processing_lambda_name
image_tag_mutability = "MUTABLE"

force_delete = true

image_scanning_configuration {
scan_on_push = true
}
}

resource "aws_codebuild_project" "viz_schism_fim_processing_lambda" {
name = local.viz_schism_fim_processing_lambda_name
description = "Codebuild project that builds the lambda container based on a zip file with lambda code and dockerfile. Also deploys a lambda function using the ECR image"
build_timeout = "60"
service_role = var.lambda_role

artifacts {
type = "NO_ARTIFACTS"
}

environment {
compute_type = "BUILD_GENERAL1_SMALL"
image = "aws/codebuild/standard:6.0"
type = "LINUX_CONTAINER"
image_pull_credentials_type = "CODEBUILD"
privileged_mode = true

environment_variable {
name = "AWS_DEFAULT_REGION"
value = var.region
}

environment_variable {
name = "AWS_ACCOUNT_ID"
value = var.account_id
}

environment_variable {
name = "IMAGE_REPO_NAME"
value = aws_ecr_repository.viz_schism_fim_processing_image.name
}

environment_variable {
name = "IMAGE_TAG"
value = var.ecr_repository_image_tag
}
}

source {
type = "S3"
location = "${aws_s3_object.schism_processing_zip_upload.bucket}/${aws_s3_object.schism_processing_zip_upload.key}"
}
}

resource "null_resource" "viz_schism_fim_processing_cluster" {
# Changes to any instance of the cluster requires re-provisioning
triggers = {
source_hash = data.archive_file.schism_processing_zip.output_md5
}

depends_on = [ aws_s3_object.schism_processing_zip_upload ]

provisioner "local-exec" {
command = "aws codebuild start-build --project-name ${aws_codebuild_project.viz_schism_fim_processing_lambda.name} --profile ${var.environment} --region ${var.region}"
}
}

resource "time_sleep" "wait_for_viz_schism_fim_processing_cluster" {
triggers = {
function_update = null_resource.viz_schism_fim_processing_cluster.triggers.source_hash
}
depends_on = [null_resource.viz_schism_fim_processing_cluster]

create_duration = "120s"
###########################
## SCHISM FIM PROCESSING ##
###########################

module "schism-fim" {
source = "./viz_schism_fim_processing"

environment = var.environment
account_id = var.account_id
region = var.region
ecr_repository_image_tag = local.ecr_repository_image_tag
codebuild_role = var.lambda_role
security_groups = var.hand_fim_processing_sgs
subnets = var.hand_fim_processing_subnets
deployment_bucket = var.deployment_bucket
profile_name = var.environment
viz_db_name = var.viz_db_name
viz_db_host = var.viz_db_host
viz_db_user_secret_string = var.viz_db_user_secret_string
default_tags = var.default_tags
}

data "aws_lambda_function" "viz_schism_fim_processing" {
function_name = local.viz_schism_fim_processing_lambda_name

depends_on = [
time_sleep.wait_for_viz_schism_fim_processing_cluster
]
}


output "hand_fim_processing" {
value = data.aws_lambda_function.viz_hand_fim_processing
}

output "schism_fim_processing" {
value = data.aws_lambda_function.viz_schism_fim_processing
output "schism_fim" {
value = module.schism-fim
}

output "optimize_rasters" {
Expand Down
56 changes: 43 additions & 13 deletions Core/LAMBDA/viz_functions/image_based/viz_schism_fim_processing/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,51 @@
FROM public.ecr.aws/lambda/python:3.9
FROM public.ecr.aws/amazonlinux/amazonlinux

# Copy function code
COPY lambda_function.py ${LAMBDA_TASK_ROOT}
COPY viz_classes.py ${LAMBDA_TASK_ROOT}
RUN dnf -y install gcc-c++ cpp sqlite-devel libtiff cmake python3 python3-pip \
python3-setuptools python3-devel openssl-devel tcl libtiff-devel libcurl-devel \
swig libpng-devel libjpeg-turbo-devel expat-devel wget tar gzip

RUN wget https://hdf-wordpress-1.s3.amazonaws.com/wp-content/uploads/manual/HDF5/HDF5_1_14_3/src/hdf5-1.14.3.tar.gz && \
tar zxvf hdf5-1.14.3.tar.gz && \
cd hdf5-1.14.3 && \
mkdir build && \
cd build && \
cmake .. && \
cmake --build . --parallel $(nproc) && \
cmake --install . --prefix /usr;

RUN wget https://download.osgeo.org/proj/proj-9.3.1.tar.gz && \
tar zxvf proj-9.3.1.tar.gz && \
cd proj-9.3.1/ && \
mkdir build && \
cd build && \
cmake .. && \
cmake --build . --parallel $(nproc) && \
cmake --install . --prefix /usr;

RUN wget https://github.com/OSGeo/gdal/releases/download/v3.8.3/gdal-3.8.3.tar.gz && \
tar xvzf gdal-3.8.3.tar.gz && \
cd gdal-3.8.3/ && \
mkdir build && \
cd build && \
cmake -DGDAL_BUILD_OPTIONAL_DRIVERS=OFF -DOGR_BUILD_OPTIONAL_DRIVERS=OFF .. && \
cmake --build . --parallel $(nproc) && \
cmake --install . --prefix /usr;

# END BASE IMAGE SETUP

# BEGIN SCRIPT SETUP

# Install the function's dependencies using file requirements.txt
# from your project folder.

COPY requirements.txt .
RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
RUN HDF5_DIR=/usr pip3 install --no-cache-dir --no-binary fiona -r requirements.txt

RUN mkdir -p /data/inputs
RUN mkdir -p /data/outpus
RUN mkdir -p /data/temp
# Copy function code
COPY process_schism_fim.py .
COPY viz_classes.py .

# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
CMD [ "lambda_function.lambda_handler" ]
ENV PYTHONUNBUFFERED=1
ENV GDAL_DATA=/usr/share/gdal/data

# Removes severity vulnerabilities
RUN yum -y update
# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
CMD [ "python3", "./process_schism_fim.py" ]
5 changes: 1 addition & 4 deletions Core/LAMBDA/viz_functions/image_based/viz_schism_fim_processing/buildspec.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,4 @@ phases:
commands:
- echo Build completed on `date`
- echo Pushing the Docker image...
- docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG
- echo Updating lambda
- npm install -g serverless
- sls deploy
- docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG
Loading