Skip to content

Commit

Permalink
Merge pull request #99 from OpenKBC/engineering_dev
Browse files Browse the repository at this point in the history 
Updated EFS setting for AWS batch
  • Loading branch information
@swiri021
swiri021 authored Nov 6, 2021
2 parents 4db8465 + 2795c42 commit bb13318
Show file tree
Hide file tree
Showing 5 changed files with 145 additions and 124 deletions.
6 changes: 5 additions & 1 deletion aws_module/batch_deployment/deg_pipeline_batch/batch_module_singleJob.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,21 @@

echo "Creating compute environment.."
aws batch create-compute-environment --compute-environment-name deg-pipeline-env \
--type MANAGED --compute-resources type=FARGATE,maxvCpus=8,securityGroupIds=sg-08946d1b26a30d376,subnets=[subnet-46231822,subnet-5c5f8b53]
--type MANAGED --compute-resources type=FARGATE,maxvCpus=8,securityGroupIds=sg-08946d1b26a30d376,subnets=[subnet-c6bdede9,subnet-23a0b868]

sleep 3

echo "Creating job queue.."
aws batch create-job-queue --job-queue-name deg-pipeline-queue --compute-environment-order order=1,computeEnvironment=deg-pipeline-env --priority 100

sleep 5

echo "Creating job.."
aws batch register-job-definition --job-definition-name deg-pipeline-job --platform-capabilities FARGATE \
--type container --container-properties file://container_configure.json

sleep 5

echo "Submit.."
aws batch submit-job --cli-input-json file://submit_configure.json > job.submitted

Expand Down
120 changes: 64 additions & 56 deletions aws_module/batch_deployment/deg_pipeline_batch/container_configure.json
View file
Original file line number Diff line number Diff line change
@@ -1,59 +1,67 @@
{
"image" : "swiri021/deg_pipeline:v1.0.0",
"jobRoleArn" : "arn:aws:iam::601333025120:role/ecsTaskExecutionRole",
"executionRoleArn" : "arn:aws:iam::601333025120:role/ecsTaskExecutionRole",

"command":[ "sh", "pipeline_controller.sh", "CD4", "Sex", "M", "F"],
"mountPoints": [
{
"sourceVolume": "efsVolume",
"containerPath": "/output",
"readOnly": false
{
"image": "swiri021/deg_pipeline:v1.0.0",
"jobRoleArn": "arn:aws:iam::601333025120:role/ecsTaskExecutionRole",
"executionRoleArn": "arn:aws:iam::601333025120:role/ecsTaskExecutionRole",
"command": [
"sh",
"pipeline_controller.sh",
"CD4",
"Sex",
"M",
"F"
],
"mountPoints": [
{
"sourceVolume": "efsVolume",
"containerPath": "/output",
"readOnly": false
}
],
"volumes": [
{
"name": "efsVolume",
"efsVolumeConfiguration": {
"fileSystemId": "fs-0d9d743a5e5d7a503",
"transitEncryption": "ENABLED",
"authorizationConfig": {
"accessPointId": "fsap-08b92207be73a6944",
"iam": "ENABLED"
}
],
"volumes": [
{
"name": "efsVolume",
"efsVolumeConfiguration": {
"fileSystemId": "fs-017866e38b27933d8"
}
}
],

"networkConfiguration": {
"assignPublicIp": "ENABLED"
}
}
],
"networkConfiguration": {
"assignPublicIp": "ENABLED"
},
"fargatePlatformConfiguration": {
"platformVersion": "1.4.0"
},
"resourceRequirements": [
{
"value": "2",
"type": "VCPU"
},

"fargatePlatformConfiguration" : {
"platformVersion": "1.4.0"
{
"value": "5120",
"type": "MEMORY"
}
],
"environment": [
{
"name": "metafile",
"value": "EPIC_HCvB_metadata_baseline_updated-share.csv"
},

"resourceRequirements" : [
{
"value":"2",
"type":"VCPU"
},
{
"value":"5120",
"type":"MEMORY"
}
],
"environment": [
{
"name": "metafile",
"value": "EPIC_HCvB_metadata_baseline_updated-share.csv"
},
{
"name": "efspoint",
"value": "/output/"
},
{
"name": "mainbucket",
"value": "openkbc-ms-maindata-bucket"
},
{
"name": "uploadbucket",
"value": "openkbc-ms-batchresult-bucket"
}
]
}
{
"name": "efspoint",
"value": "/output/"
},
{
"name": "mainbucket",
"value": "openkbc-ms-maindata-bucket"
},
{
"name": "uploadbucket",
"value": "openkbc-ms-batchresult-bucket"
}
]
}
9 changes: 7 additions & 2 deletions aws_module/batch_deployment/feature_extraction_batch/batch_module_singleJob.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,27 @@

## Need to push job docker images before running this module, this module is an example for how to run single job for AWS batch.
## This is same pipeline with snakemake pipeline and it will generate DEG result for one cell type.

# echo "Creating EFS to share files.."


echo "Creating compute environment.."
aws batch create-compute-environment --compute-environment-name feature-ext-pipeline-env \
--type MANAGED --compute-resources type=FARGATE,maxvCpus=8,securityGroupIds=sg-08946d1b26a30d376,subnets=[subnet-46231822,subnet-5c5f8b53]
--type MANAGED --compute-resources type=FARGATE,maxvCpus=8,securityGroupIds=sg-08946d1b26a30d376,subnets=[subnet-c6bdede9,subnet-23a0b868]

sleep 3
sleep 5

echo "Creating job queue.."
aws batch create-job-queue --job-queue-name feature-ext-pipeline-queue --compute-environment-order order=1,computeEnvironment=feature-ext-pipeline-env --priority 100

sleep 5

echo "Creating job.."
aws batch register-job-definition --job-definition-name feature-ext-pipeline-job --platform-capabilities FARGATE \
--type container --container-properties file://container_configure.json

sleep 5

echo "Submit.."
aws batch submit-job --cli-input-json file://submit_configure.json > job.submitted

Expand Down
132 changes: 68 additions & 64 deletions aws_module/batch_deployment/feature_extraction_batch/container_configure.json
View file
Original file line number Diff line number Diff line change
@@ -1,67 +1,71 @@
{
"image" : "swiri021/deg_pipeline:v1.0.0",
"jobRoleArn" : "arn:aws:iam::601333025120:role/ecsTaskExecutionRole",
"executionRoleArn" : "arn:aws:iam::601333025120:role/ecsTaskExecutionRole",

"command":[ "sh", "pipeline_controller.sh"],
"mountPoints": [
{
"sourceVolume": "efsVolume",
"containerPath": "/output",
"readOnly": false
{
"image": "swiri021/feature_ext_pipeline:v1.0.0",
"jobRoleArn": "arn:aws:iam::601333025120:role/ecsTaskExecutionRole",
"executionRoleArn": "arn:aws:iam::601333025120:role/ecsTaskExecutionRole",
"command": [
"sh",
"pipeline_controller.sh"
],
"mountPoints": [
{
"sourceVolume": "efsVolume",
"containerPath": "/output",
"readOnly": false
}
],
"volumes": [
{
"name": "efsVolume",
"efsVolumeConfiguration": {
"fileSystemId": "fs-0d9d743a5e5d7a503",
"transitEncryption": "ENABLED",
"authorizationConfig": {
"accessPointId": "fsap-0be7f3243cac70a52",
"iam": "ENABLED"
}
],
"volumes": [
{
"name": "efsVolume",
"efsVolumeConfiguration": {
"fileSystemId": "fs-017866e38b27933d8"
}
}
],

"networkConfiguration": {
"assignPublicIp": "ENABLED"
}
}
],
"networkConfiguration": {
"assignPublicIp": "ENABLED"
},
"fargatePlatformConfiguration": {
"platformVersion": "1.4.0"
},
"resourceRequirements": [
{
"value": "2",
"type": "VCPU"
},

"fargatePlatformConfiguration" : {
"platformVersion": "1.4.0"
{
"value": "5120",
"type": "MEMORY"
}
],
"environment": [
{
"name": "metafile",
"value": "EPIC_HCvB_metadata_baseline_updated-share.csv"
},

"resourceRequirements" : [
{
"value":"2",
"type":"VCPU"
},
{
"value":"5120",
"type":"MEMORY"
}
],
"environment": [
{
"name": "metafile",
"value": "EPIC_HCvB_metadata_baseline_updated-share.csv"
},
{
"name": "efspoint",
"value": "/output/"
},
{
"name": "msigDBPATH",
"value": "msigdb.v7.4.entrez.gmt"
},
{
"name": "startFile",
"value": "counts_vst_CD4.csv"
},
{
"name": "mainbucket",
"value": "openkbc-ms-maindata-bucket"
},
{
"name": "uploadbucket",
"value": "openkbc-ms-batchresult-bucket"
}
]
}
{
"name": "efspoint",
"value": "/output/"
},
{
"name": "msigDBPATH",
"value": "msigdb.v7.4.entrez.gmt"
},
{
"name": "startFile",
"value": "counts_vst_CD4.csv"
},
{
"name": "mainbucket",
"value": "openkbc-ms-maindata-bucket"
},
{
"name": "uploadbucket",
"value": "openkbc-ms-batchresult-bucket"
}
]
}
2 changes: 1 addition & 1 deletion aws_module/batch_deployment/feature_extraction_batch/submit_configure.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"jobName": "feature-ext-pipeline-job",
"jobQueue": "feature-ext-pipeline-queue",
"arrayProperties": {
"size": 3
"size": 9
},
"dependsOn":[
{
Expand Down

0 comments on commit bb13318

Please sign in to comment.