From 3deb439fcd96e83d18090bdbf43aa442cebbfe02 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Wed, 16 Nov 2022 14:41:05 -0600 Subject: [PATCH 1/2] Create image_build_push.yaml (#41) --- .github/workflows/image_build_push.yaml | 71 +++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/image_build_push.yaml diff --git a/.github/workflows/image_build_push.yaml b/.github/workflows/image_build_push.yaml new file mode 100644 index 0000000..524d2b8 --- /dev/null +++ b/.github/workflows/image_build_push.yaml @@ -0,0 +1,71 @@ +name: Build Image and Push + +on: push + +jobs: + manifest-indexing: + uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master + with: + OVERRIDE_REPO_NAME: "manifest-indexing" + DOCKERFILE_LOCATION: "./manifest_indexing/manifest_indexing.Dockerfile" + DOCKERFILE_BUILD_CONTEXT: "./manifest_indexing" + secrets: + ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} + ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + download-indexd-manifest: + uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master + with: + OVERRIDE_REPO_NAME: "download-indexd-manifest" + DOCKERFILE_LOCATION: "./manifest_indexing/indexd_manifest_job.Dockerfile" + DOCKERFILE_BUILD_CONTEXT: "./manifest_indexing" + secrets: + ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} + ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + manifest-merging: + uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master + with: + OVERRIDE_REPO_NAME: "manifest-merging" + DOCKERFILE_LOCATION: "./manifest_merging/manifest_merging.Dockerfile" + DOCKERFILE_BUILD_CONTEXT: "./manifest_merging" + secrets: + ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} + ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + metadata-manifest-ingestion: + uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master + with: + OVERRIDE_REPO_NAME: "metadata-manifest-ingestion" + DOCKERFILE_LOCATION: "./metadata_ingestion/metadata_ingestion.Dockerfile" + DOCKERFILE_BUILD_CONTEXT: "./metadata_ingestion" + secrets: + ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} + ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + get-dbgap-metadata: + uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master + with: + OVERRIDE_REPO_NAME: "get-dbgap-metadata" + DOCKERFILE_LOCATION: "./metadata_ingestion/get_dbgap_metadata_manifest.Dockerfile" + DOCKERFILE_BUILD_CONTEXT: "./metadata_ingestion" + secrets: + ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} + ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + batch-export: + uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master + with: + OVERRIDE_REPO_NAME: "batch-export" + DOCKERFILE_LOCATION: "./export_job/Dockerfile" + DOCKERFILE_BUILD_CONTEXT: "./export_job" + secrets: + ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} + ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} From d4393a93748f7b0ec579ddb22c8427ded4139ca5 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:04:12 -0700 Subject: [PATCH 2/2] Merge pull request #53 from uc-cdis/feat/GPE-1219 Export Job to IRSA --- export_job/export_zip_from_manifest_job.py | 31 ++++++++-------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/export_job/export_zip_from_manifest_job.py b/export_job/export_zip_from_manifest_job.py index b59cda2..9be7660 100644 --- a/export_job/export_zip_from_manifest_job.py +++ b/export_job/export_zip_from_manifest_job.py @@ -93,19 +93,12 @@ def download_files(access_token, hostname): shutil.make_archive(EXPORT_DIR, "zip", EXPORT_DIR) -def upload_export_to_s3(s3_credentials, username): +def upload_export_to_s3(bucket_name, username): """ - uploads the local zip export to s3 and returns a presigned url, expires after 1 hour + Uploads the local zip export to S3 and returns a presigned URL, expires after 1 hour. """ - bucket_name = s3_credentials["bucket_name"] - aws_access_key_id = s3_credentials["aws_access_key_id"] - aws_secret_access_key = s3_credentials["aws_secret_access_key"] - - s3_client = boto3.client( - "s3", - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - ) + + s3_client = boto3.client("s3") export_key = f"{quote_plus(username)}-export.zip" s3_client.upload_file("export.zip", bucket_name, export_key) @@ -113,6 +106,7 @@ def upload_export_to_s3(s3_credentials, username): url = s3_client.generate_presigned_url( "get_object", Params={"Bucket": bucket_name, "Key": export_key}, ExpiresIn=3600 ) + return url @@ -129,7 +123,11 @@ def fail(error_message=DEFAULT_ERROR_MESSAGE): if __name__ == "__main__": access_token = os.environ["ACCESS_TOKEN"] hostname = os.environ["GEN3_HOSTNAME"] - + try: + bucket_name = os.environ["BUCKET"] + except Exception as e: + print(f"Error: Environment variable 'BUCKET' is not set.") + fail() try: input_data = json.loads(os.environ["INPUT_DATA"]) except Exception as e: @@ -144,13 +142,6 @@ def fail(error_message=DEFAULT_ERROR_MESSAGE): "No studies or files provided. Please select some studies or files and try again." ) - try: - with open("/batch-export-creds.json") as creds_file: - s3_credentials = json.load(creds_file) - except: - print("S3 is misconfigured for this job.") - fail() - try: username_req = requests.get( f"https://{hostname}/user/user", @@ -181,7 +172,7 @@ def fail(error_message=DEFAULT_ERROR_MESSAGE): fail() try: - presigned_url = upload_export_to_s3(s3_credentials, username) + presigned_url = upload_export_to_s3(bucket_name, username) except Exception as e: print(f"Export to s3 failed {repr(e)}") fail()