From 6d0dfa4a250c7b8e2173ada2d2dc0a80f05a38ec Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Thu, 4 Apr 2024 11:57:16 -0700 Subject: [PATCH 1/5] Create notebook to update collection hrefs --- transformation-scripts/update-hrefs.ipynb | 150 ++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 transformation-scripts/update-hrefs.ipynb diff --git a/transformation-scripts/update-hrefs.ipynb b/transformation-scripts/update-hrefs.ipynb new file mode 100644 index 00000000..88cff913 --- /dev/null +++ b/transformation-scripts/update-hrefs.ipynb @@ -0,0 +1,150 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Notebook to update hrefs in particular collections" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import boto3\n", + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "AWS_ACCESS_KEY_ID = \"[CHANGE ME]\"\n", + "AWS_SECRET_ACCESS_KEY = \"[CHANGE ME]\"\n", + "AWS_SESSION_TOKEN = \"[CHANGE ME]\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "s3_client = boto3.client(\n", + " \"s3\",\n", + " aws_access_key_id=AWS_ACCESS_KEY_ID,\n", + " aws_secret_access_key=AWS_SECRET_ACCESS_KEY,\n", + " aws_session_token=AWS_SESSION_TOKEN,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `update_json_href` function takes in a bucket_name, s3_prefix, old_href_substring and new_href_substring" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def update_json_href(\n", + " bucket_name, collection_name, old_href_substring, new_href_substring\n", + "):\n", + " \"\"\"Given the bucket name, s3 prefix,\n", + " update all hrefs in the path bucketname/s3_prefix\n", + " and update all the old_href_substring to be new_href_substring.\n", + "\n", + " Keyword arguments:\n", + " bucket_name -- the s3 bucket name\n", + " collection_name -- the collection name\n", + " old_href_substring -- the string to replace in href\n", + " new_href_substring -- the new href substring\n", + " \"\"\"\n", + " s3 = s3_client\n", + " s3_prefix = f\"{collection_name}/\"\n", + "\n", + " response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_prefix)\n", + "\n", + " for obj in response.get(\"Contents\", []):\n", + " key = obj[\"Key\"]\n", + "\n", + " # Check if the object is a JSON file\n", + " if key.endswith(\".json\"):\n", + " response = s3.get_object(Bucket=bucket_name, Key=key)\n", + " json_data = response[\"Body\"].read().decode(\"utf-8\")\n", + "\n", + " data = json.loads(json_data)\n", + " for assets_key in data[\"assets\"]:\n", + " # Update href property\n", + " data[\"assets\"][assets_key][\"href\"] = data[\"assets\"][assets_key][\n", + " \"href\"\n", + " ].replace(old_href_substring, new_href_substring)\n", + " # Serialize updated JSON\n", + " updated_json = json.dumps(data)\n", + "\n", + " # Upload updated JSON file back to S3\n", + " s3.put_object(Bucket=bucket_name, Key=key, Body=updated_json)\n", + "\n", + " print(f\"Updated {key}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The next cell calls update_json_href to update the hlsl30-ej-reprocessed and hlss30-ej-reprocessed collections from the veda-data-store bucket. Specifically, it updates the href instances of \"covid-eo-data\" to \"veda-data-store\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "update_json_href(\n", + " \"veda-data-store\", \"hlsl30-ej-reprocessed\", \"covid-eo-data\", \"veda-data-store\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "update_json_href(\n", + " \"veda-data-store\", \"hlss30-ej-reprocessed\", \"covid-eo-data\", \"veda-data-store\"\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From a68750be202cf791018292f61b1c732aed80102a Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Thu, 4 Apr 2024 15:53:16 -0700 Subject: [PATCH 2/5] Simplify notebook to get json_keys only --- transformation-scripts/update-hrefs.ipynb | 34 +++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/transformation-scripts/update-hrefs.ipynb b/transformation-scripts/update-hrefs.ipynb index 88cff913..d06e8c22 100644 --- a/transformation-scripts/update-hrefs.ipynb +++ b/transformation-scripts/update-hrefs.ipynb @@ -73,27 +73,27 @@ "\n", " response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_prefix)\n", "\n", - " for obj in response.get(\"Contents\", []):\n", - " key = obj[\"Key\"]\n", + " json_keys = [\n", + " item[\"Key\"] for item in response[\"Contents\"] if item[\"Key\"].endswith(\".json\")\n", + " ]\n", "\n", - " # Check if the object is a JSON file\n", - " if key.endswith(\".json\"):\n", - " response = s3.get_object(Bucket=bucket_name, Key=key)\n", - " json_data = response[\"Body\"].read().decode(\"utf-8\")\n", + " for key in json_keys:\n", + " response = s3.get_object(Bucket=bucket_name, Key=key)\n", + " json_data = response[\"Body\"].read().decode(\"utf-8\")\n", "\n", - " data = json.loads(json_data)\n", - " for assets_key in data[\"assets\"]:\n", - " # Update href property\n", - " data[\"assets\"][assets_key][\"href\"] = data[\"assets\"][assets_key][\n", - " \"href\"\n", - " ].replace(old_href_substring, new_href_substring)\n", - " # Serialize updated JSON\n", - " updated_json = json.dumps(data)\n", + " data = json.loads(json_data)\n", + " for assets_key in data[\"assets\"]:\n", + " # Update href property\n", + " data[\"assets\"][assets_key][\"href\"] = data[\"assets\"][assets_key][\n", + " \"href\"\n", + " ].replace(old_href_substring, new_href_substring)\n", + " # Serialize updated JSON\n", + " updated_json = json.dumps(data)\n", "\n", - " # Upload updated JSON file back to S3\n", - " s3.put_object(Bucket=bucket_name, Key=key, Body=updated_json)\n", + " # Upload updated JSON file back to S3\n", + " s3.put_object(Bucket=bucket_name, Key=key, Body=updated_json)\n", "\n", - " print(f\"Updated {key}\")" + " print(f\"Updated {key}\")" ] }, { From 925d17cc92ed0da81a903f104cf88d6e5d00f959 Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Fri, 5 Apr 2024 13:07:14 -0700 Subject: [PATCH 3/5] Move put object call to outside of loop --- transformation-scripts/update-hrefs.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/transformation-scripts/update-hrefs.ipynb b/transformation-scripts/update-hrefs.ipynb index d06e8c22..1b62a28f 100644 --- a/transformation-scripts/update-hrefs.ipynb +++ b/transformation-scripts/update-hrefs.ipynb @@ -82,17 +82,17 @@ " json_data = response[\"Body\"].read().decode(\"utf-8\")\n", "\n", " data = json.loads(json_data)\n", - " for assets_key in data[\"assets\"]:\n", + " for assets_key in data[\"assets\"]: # the condition in question for looping\n", " # Update href property\n", " data[\"assets\"][assets_key][\"href\"] = data[\"assets\"][assets_key][\n", " \"href\"\n", " ].replace(old_href_substring, new_href_substring)\n", - " # Serialize updated JSON\n", - " updated_json = json.dumps(data)\n", "\n", - " # Upload updated JSON file back to S3\n", - " s3.put_object(Bucket=bucket_name, Key=key, Body=updated_json)\n", + " # Serialize updated JSON\n", + " updated_json = json.dumps(data)\n", "\n", + " # Upload updated JSON file back to S3, commented out the line below so it doesn't actually upload\n", + " s3.put_object(Bucket=bucket_name, Key=key, Body=updated_json)\n", " print(f\"Updated {key}\")" ] }, From 034d3750012524bae8bb0819dadb4b06b60b2038 Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Fri, 5 Apr 2024 13:11:06 -0700 Subject: [PATCH 4/5] Remove unnecessary comment --- transformation-scripts/update-hrefs.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformation-scripts/update-hrefs.ipynb b/transformation-scripts/update-hrefs.ipynb index 1b62a28f..1f1edcc9 100644 --- a/transformation-scripts/update-hrefs.ipynb +++ b/transformation-scripts/update-hrefs.ipynb @@ -82,7 +82,7 @@ " json_data = response[\"Body\"].read().decode(\"utf-8\")\n", "\n", " data = json.loads(json_data)\n", - " for assets_key in data[\"assets\"]: # the condition in question for looping\n", + " for assets_key in data[\"assets\"]:\n", " # Update href property\n", " data[\"assets\"][assets_key][\"href\"] = data[\"assets\"][assets_key][\n", " \"href\"\n", From 84c273f10a524429669db4a661af22b410b9afb8 Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Mon, 8 Apr 2024 09:58:22 -0700 Subject: [PATCH 5/5] Add output from cells that were run --- transformation-scripts/update-hrefs.ipynb | 72 ++++++++++++++++++++--- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/transformation-scripts/update-hrefs.ipynb b/transformation-scripts/update-hrefs.ipynb index 1f1edcc9..b3d57b47 100644 --- a/transformation-scripts/update-hrefs.ipynb +++ b/transformation-scripts/update-hrefs.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 106, "metadata": {}, "outputs": [], "source": [ @@ -19,7 +19,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 107, "metadata": {}, "outputs": [], "source": [ @@ -30,7 +30,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 108, "metadata": {}, "outputs": [], "source": [ @@ -51,7 +51,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 109, "metadata": {}, "outputs": [], "source": [ @@ -105,9 +105,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 110, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017157T144341.v2.0/HLS.L30.T19QHA.2017157T144341.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017173T144347.v2.0/HLS.L30.T19QHA.2017173T144347.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017205T144356.v2.0/HLS.L30.T19QHA.2017205T144356.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017221T144403.v2.0/HLS.L30.T19QHA.2017221T144403.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017237T144407.v2.0/HLS.L30.T19QHA.2017237T144407.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017269T144414.v2.0/HLS.L30.T19QHA.2017269T144414.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017285T144419.v2.0/HLS.L30.T19QHA.2017285T144419.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017301T144420.v2.0/HLS.L30.T19QHA.2017301T144420.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017317T144417.v2.0/HLS.L30.T19QHA.2017317T144417.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017333T144411.v2.0/HLS.L30.T19QHA.2017333T144411.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021182T163159.v2.0/HLS.L30.T15RYP.2021182T163159.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021198T163201.v2.0/HLS.L30.T15RYP.2021198T163201.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021230T163215.v2.0/HLS.L30.T15RYP.2021230T163215.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021246T163220.v2.0/HLS.L30.T15RYP.2021246T163220.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021262T163223.v2.0/HLS.L30.T15RYP.2021262T163223.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021278T163229.v2.0/HLS.L30.T15RYP.2021278T163229.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021294T163232.v2.0/HLS.L30.T15RYP.2021294T163232.v2.0_stac-ej-reprocessed.json\n" + ] + } + ], "source": [ "update_json_href(\n", " \"veda-data-store\", \"hlsl30-ej-reprocessed\", \"covid-eo-data\", \"veda-data-store\"\n", @@ -116,9 +140,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 111, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017193T150719.v2.0/HLS.S30.T19QHA.2017193T150719.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017218T150721.v2.0/HLS.S30.T19QHA.2017218T150721.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017233T150719.v2.0/HLS.S30.T19QHA.2017233T150719.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017278T150721.v2.0/HLS.S30.T19QHA.2017278T150721.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017293T150709.v2.0/HLS.S30.T19QHA.2017293T150709.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017318T150721.v2.0/HLS.S30.T19QHA.2017318T150721.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017323T150709.v2.0/HLS.S30.T19QHA.2017323T150709.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017333T150709.v2.0/HLS.S30.T19QHA.2017333T150709.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021185T163839.v2.0/HLS.S30.T15RYP.2021185T163839.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021195T163839.v2.0/HLS.S30.T15RYP.2021195T163839.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021200T163901.v2.0/HLS.S30.T15RYP.2021200T163901.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021205T163839.v2.0/HLS.S30.T15RYP.2021205T163839.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021210T163901.v2.0/HLS.S30.T15RYP.2021210T163901.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021220T163901.v2.0/HLS.S30.T15RYP.2021220T163901.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021225T163839.v2.0/HLS.S30.T15RYP.2021225T163839.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021235T163839.v2.0/HLS.S30.T15RYP.2021235T163839.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021245T163839.v2.0/HLS.S30.T15RYP.2021245T163839.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021250T163901.v2.0/HLS.S30.T15RYP.2021250T163901.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021265T163909.v2.0/HLS.S30.T15RYP.2021265T163909.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021270T164051.v2.0/HLS.S30.T15RYP.2021270T164051.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021280T164201.v2.0/HLS.S30.T15RYP.2021280T164201.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021285T164139.v2.0/HLS.S30.T15RYP.2021285T164139.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021290T164311.v2.0/HLS.S30.T15RYP.2021290T164311.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021295T164339.v2.0/HLS.S30.T15RYP.2021295T164339.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021300T164411.v2.0/HLS.S30.T15RYP.2021300T164411.v2.0_stac-ej-reprocessed.json\n" + ] + } + ], "source": [ "update_json_href(\n", " \"veda-data-store\", \"hlss30-ej-reprocessed\", \"covid-eo-data\", \"veda-data-store\"\n",