diff --git a/transformation-scripts/update-hrefs.ipynb b/transformation-scripts/update-hrefs.ipynb new file mode 100644 index 00000000..b3d57b47 --- /dev/null +++ b/transformation-scripts/update-hrefs.ipynb @@ -0,0 +1,206 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Notebook to update hrefs in particular collections" + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "metadata": {}, + "outputs": [], + "source": [ + "import boto3\n", + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": {}, + "outputs": [], + "source": [ + "AWS_ACCESS_KEY_ID = \"[CHANGE ME]\"\n", + "AWS_SECRET_ACCESS_KEY = \"[CHANGE ME]\"\n", + "AWS_SESSION_TOKEN = \"[CHANGE ME]\"" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [], + "source": [ + "s3_client = boto3.client(\n", + " \"s3\",\n", + " aws_access_key_id=AWS_ACCESS_KEY_ID,\n", + " aws_secret_access_key=AWS_SECRET_ACCESS_KEY,\n", + " aws_session_token=AWS_SESSION_TOKEN,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `update_json_href` function takes in a bucket_name, s3_prefix, old_href_substring and new_href_substring" + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "metadata": {}, + "outputs": [], + "source": [ + "def update_json_href(\n", + " bucket_name, collection_name, old_href_substring, new_href_substring\n", + "):\n", + " \"\"\"Given the bucket name, s3 prefix,\n", + " update all hrefs in the path bucketname/s3_prefix\n", + " and update all the old_href_substring to be new_href_substring.\n", + "\n", + " Keyword arguments:\n", + " bucket_name -- the s3 bucket name\n", + " collection_name -- the collection name\n", + " old_href_substring -- the string to replace in href\n", + " new_href_substring -- the new href substring\n", + " \"\"\"\n", + " s3 = s3_client\n", + " s3_prefix = f\"{collection_name}/\"\n", + "\n", + " response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_prefix)\n", + "\n", + " json_keys = [\n", + " item[\"Key\"] for item in response[\"Contents\"] if item[\"Key\"].endswith(\".json\")\n", + " ]\n", + "\n", + " for key in json_keys:\n", + " response = s3.get_object(Bucket=bucket_name, Key=key)\n", + " json_data = response[\"Body\"].read().decode(\"utf-8\")\n", + "\n", + " data = json.loads(json_data)\n", + " for assets_key in data[\"assets\"]:\n", + " # Update href property\n", + " data[\"assets\"][assets_key][\"href\"] = data[\"assets\"][assets_key][\n", + " \"href\"\n", + " ].replace(old_href_substring, new_href_substring)\n", + "\n", + " # Serialize updated JSON\n", + " updated_json = json.dumps(data)\n", + "\n", + " # Upload updated JSON file back to S3, commented out the line below so it doesn't actually upload\n", + " s3.put_object(Bucket=bucket_name, Key=key, Body=updated_json)\n", + " print(f\"Updated {key}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The next cell calls update_json_href to update the hlsl30-ej-reprocessed and hlss30-ej-reprocessed collections from the veda-data-store bucket. Specifically, it updates the href instances of \"covid-eo-data\" to \"veda-data-store\"" + ] + }, + { + "cell_type": "code", + "execution_count": 110, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017157T144341.v2.0/HLS.L30.T19QHA.2017157T144341.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017173T144347.v2.0/HLS.L30.T19QHA.2017173T144347.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017205T144356.v2.0/HLS.L30.T19QHA.2017205T144356.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017221T144403.v2.0/HLS.L30.T19QHA.2017221T144403.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017237T144407.v2.0/HLS.L30.T19QHA.2017237T144407.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017269T144414.v2.0/HLS.L30.T19QHA.2017269T144414.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017285T144419.v2.0/HLS.L30.T19QHA.2017285T144419.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017301T144420.v2.0/HLS.L30.T19QHA.2017301T144420.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017317T144417.v2.0/HLS.L30.T19QHA.2017317T144417.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2017/19QHA/HLS.L30.T19QHA.2017333T144411.v2.0/HLS.L30.T19QHA.2017333T144411.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021182T163159.v2.0/HLS.L30.T15RYP.2021182T163159.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021198T163201.v2.0/HLS.L30.T15RYP.2021198T163201.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021230T163215.v2.0/HLS.L30.T15RYP.2021230T163215.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021246T163220.v2.0/HLS.L30.T15RYP.2021246T163220.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021262T163223.v2.0/HLS.L30.T15RYP.2021262T163223.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021278T163229.v2.0/HLS.L30.T15RYP.2021278T163229.v2.0_stac-ej-reprocessed.json\n", + "Updated hlsl30-ej-reprocessed/2021/15RYP/HLS.L30.T15RYP.2021294T163232.v2.0/HLS.L30.T15RYP.2021294T163232.v2.0_stac-ej-reprocessed.json\n" + ] + } + ], + "source": [ + "update_json_href(\n", + " \"veda-data-store\", \"hlsl30-ej-reprocessed\", \"covid-eo-data\", \"veda-data-store\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 111, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017193T150719.v2.0/HLS.S30.T19QHA.2017193T150719.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017218T150721.v2.0/HLS.S30.T19QHA.2017218T150721.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017233T150719.v2.0/HLS.S30.T19QHA.2017233T150719.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017278T150721.v2.0/HLS.S30.T19QHA.2017278T150721.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017293T150709.v2.0/HLS.S30.T19QHA.2017293T150709.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017318T150721.v2.0/HLS.S30.T19QHA.2017318T150721.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017323T150709.v2.0/HLS.S30.T19QHA.2017323T150709.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2017/19QHA/HLS.S30.T19QHA.2017333T150709.v2.0/HLS.S30.T19QHA.2017333T150709.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021185T163839.v2.0/HLS.S30.T15RYP.2021185T163839.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021195T163839.v2.0/HLS.S30.T15RYP.2021195T163839.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021200T163901.v2.0/HLS.S30.T15RYP.2021200T163901.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021205T163839.v2.0/HLS.S30.T15RYP.2021205T163839.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021210T163901.v2.0/HLS.S30.T15RYP.2021210T163901.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021220T163901.v2.0/HLS.S30.T15RYP.2021220T163901.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021225T163839.v2.0/HLS.S30.T15RYP.2021225T163839.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021235T163839.v2.0/HLS.S30.T15RYP.2021235T163839.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021245T163839.v2.0/HLS.S30.T15RYP.2021245T163839.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021250T163901.v2.0/HLS.S30.T15RYP.2021250T163901.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021265T163909.v2.0/HLS.S30.T15RYP.2021265T163909.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021270T164051.v2.0/HLS.S30.T15RYP.2021270T164051.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021280T164201.v2.0/HLS.S30.T15RYP.2021280T164201.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021285T164139.v2.0/HLS.S30.T15RYP.2021285T164139.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021290T164311.v2.0/HLS.S30.T15RYP.2021290T164311.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021295T164339.v2.0/HLS.S30.T15RYP.2021295T164339.v2.0_stac-ej-reprocessed.json\n", + "Updated hlss30-ej-reprocessed/2021/15RYP/HLS.S30.T15RYP.2021300T164411.v2.0/HLS.S30.T15RYP.2021300T164411.v2.0_stac-ej-reprocessed.json\n" + ] + } + ], + "source": [ + "update_json_href(\n", + " \"veda-data-store\", \"hlss30-ej-reprocessed\", \"covid-eo-data\", \"veda-data-store\"\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}