From f7340ee2d94c3695728dc86dc64642f2d2137787 Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Wed, 12 Feb 2025 16:41:45 -0800 Subject: [PATCH 01/13] fix: create temporary directory for artifact uploading and handle multiple files for promotion process (#269) --- .github/workflows/pr.yml | 20 ++++- .../dataset-config/test-changed-files.json | 90 +++++++++++++++++++ .../dataset-config/test-changed-files-2.json | 90 +++++++++++++++++++ .../dataset-config/test-changed-files.json | 90 +++++++++++++++++++ .../staging/dataset-config/test-run-id.json | 0 .../staging/dataset-config/test.json | 90 ------------------- scripts/promote_to_production.py | 1 + 7 files changed, 287 insertions(+), 94 deletions(-) create mode 100644 ingestion-data/production/dataset-config/test-changed-files.json create mode 100644 ingestion-data/staging/dataset-config/test-changed-files-2.json create mode 100644 ingestion-data/staging/dataset-config/test-changed-files.json delete mode 100644 ingestion-data/staging/dataset-config/test-run-id.json diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index aa15228d..ff39dd7c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -72,8 +72,7 @@ jobs: id: changed-files uses: tj-actions/changed-files@v45 with: - files: | - **.json + files: ingestion-data/staging/dataset-config/**.json - name: List all newly added files env: @@ -238,14 +237,27 @@ jobs: echo "Final collection_ids: $collection_ids" echo "collection_ids=${collection_ids}" >> $GITHUB_ENV + - name: Prepare temp directory for artifacts + run: | + mkdir -p /tmp/upload-artifacts + for file in $(echo "${{ needs.publish-new-datasets.outputs.publishedCollections }}" | tr ',' ' '); do + if [ -f "$file" ]; then + cp "$file" /tmp/upload-artifacts/ + else + echo "Warning: File not found - $file" + fi + done + + - name: List files in temp directory + run: ls -lah /tmp/upload-artifacts + - name: Upload published collections files as directory artifact uses: actions/upload-artifact@v4 env: - PUBLISHED_COLLECTION_FILES: ${{ needs.publish-new-datasets.outputs.publishedCollections }} ARTIFACT_RETENTION_DAYS: ${{ vars.ARTIFACT_RETENTION_DAYS }} with: name: collections-to-promote-from-${{ github.event.pull_request.number }} - path: ${{ env.PUBLISHED_COLLECTION_FILES }} + path: /tmp/upload-artifacts/* retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} - name: Save Workflow Run ID diff --git a/ingestion-data/production/dataset-config/test-changed-files.json b/ingestion-data/production/dataset-config/test-changed-files.json new file mode 100644 index 00000000..f658515a --- /dev/null +++ b/ingestion-data/production/dataset-config/test-changed-files.json @@ -0,0 +1,90 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00.000Z", + "enddate": "2021-09-09T23:59:59.000Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)NDWI_Difference_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + }, + "collection": "ida-ndwi-difference-TEST-SHOULDNT-RUN", + "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-SHOULDNT-RUN", + "dashboard:time_density": "day", + "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.", + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [ + 1 + ], + "colormap_name": "rdbu", + "assets": [ + "cog_default" + ], + "rescale": [ + [ + -1, + 1 + ] + ], + "title": "VEDA Dashboard Render Parameters" + } + }, + "dashboard:is_periodic": true + } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/test-changed-files-2.json b/ingestion-data/staging/dataset-config/test-changed-files-2.json new file mode 100644 index 00000000..22e33065 --- /dev/null +++ b/ingestion-data/staging/dataset-config/test-changed-files-2.json @@ -0,0 +1,90 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00.000Z", + "enddate": "2021-09-09T23:59:59.000Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)NDWI_Difference_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + }, + "collection": "ida-ndwi-difference-TEST-2", + "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-2", + "dashboard:time_density": "day", + "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.", + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [ + 1 + ], + "colormap_name": "rdbu", + "assets": [ + "cog_default" + ], + "rescale": [ + [ + -1, + 1 + ] + ], + "title": "VEDA Dashboard Render Parameters" + } + }, + "dashboard:is_periodic": true + } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/test-changed-files.json b/ingestion-data/staging/dataset-config/test-changed-files.json new file mode 100644 index 00000000..af614ab1 --- /dev/null +++ b/ingestion-data/staging/dataset-config/test-changed-files.json @@ -0,0 +1,90 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00.000Z", + "enddate": "2021-09-09T23:59:59.000Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)NDWI_Difference_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + }, + "collection": "ida-ndwi-difference-TEST", + "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST", + "dashboard:time_density": "day", + "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.", + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [ + 1 + ], + "colormap_name": "rdbu", + "assets": [ + "cog_default" + ], + "rescale": [ + [ + -1, + 1 + ] + ], + "title": "VEDA Dashboard Render Parameters" + } + }, + "dashboard:is_periodic": true + } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/test-run-id.json b/ingestion-data/staging/dataset-config/test-run-id.json deleted file mode 100644 index e69de29b..00000000 diff --git a/ingestion-data/staging/dataset-config/test.json b/ingestion-data/staging/dataset-config/test.json index af614ab1..e69de29b 100644 --- a/ingestion-data/staging/dataset-config/test.json +++ b/ingestion-data/staging/dataset-config/test.json @@ -1,90 +0,0 @@ -{ - "license": "CC0-1.0", - "stac_version": "1.0.0", - "links": [], - "spatial_extent": { - "xmin": -90.85, - "ymin": 29.03, - "xmax": -90.06, - "ymax": 29.65 - }, - "temporal_extent": { - "startdate": "2021-08-23T00:00:00.000Z", - "enddate": "2021-09-09T23:59:59.000Z" - }, - "discovery_items": [ - { - "upload": false, - "cogify": false, - "dry_run": false, - "filename_regex": "(.*)NDWI_Difference_(.*).tif$", - "use_multithreading": false, - "discovery": "s3", - "prefix": "planet-indices-v2/", - "bucket": "veda-data-store-staging" - } - ], - "sample_files": [ - "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif" - ], - "data_type": "cog", - "stac_extensions": [ - "https://stac-extensions.github.io/render/v1.0.0/schema.json", - "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" - ], - "item_assets": { - "cog_default": { - "type": "image/tiff; application=geotiff; profile=cloud-optimized", - "roles": [ - "data", - "layer" - ], - "title": "Default COG Layer", - "description": "Cloud optimized default layer to display on map" - } - }, - "providers": [ - { - "name": "NASA VEDA", - "roles": [ - "host" - ], - "url": "https://www.earthdata.nasa.gov/dashboard/" - } - ], - "assets": { - "thumbnail": { - "title": "Thumbnail", - "type": "image/jpeg", - "roles": [ - "thumbnail" - ], - "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", - "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" - } - }, - "collection": "ida-ndwi-difference-TEST", - "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST", - "dashboard:time_density": "day", - "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.", - "renders": { - "dashboard": { - "resampling": "nearest", - "bidx": [ - 1 - ], - "colormap_name": "rdbu", - "assets": [ - "cog_default" - ], - "rescale": [ - [ - -1, - 1 - ] - ], - "title": "VEDA Dashboard Render Parameters" - } - }, - "dashboard:is_periodic": true - } \ No newline at end of file diff --git a/scripts/promote_to_production.py b/scripts/promote_to_production.py index 1eb2197b..9b618966 100644 --- a/scripts/promote_to_production.py +++ b/scripts/promote_to_production.py @@ -74,6 +74,7 @@ def promote_to_production(payload): try: with open(sys.argv[1], "r") as file: input = json.load(file) + input["transfer"] = True discovery_items = input.get("discovery_items") validated_discovery_items = [ validate_discovery_item_config(item) for item in discovery_items From c137991f3242154eafee8dc8df3c5d95955dc848 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 12 Feb 2025 17:49:54 -0700 Subject: [PATCH 02/13] feat(datasets): add 6 salt marsh story dataset configs (#217) --- .../promotion-config/ida-ndvi-difference.json | 82 ++++++++++++++++++ .../production/promotion-config/ida-ndvi.json | 84 +++++++++++++++++++ .../promotion-config/ida-ndwi-difference.json | 82 ++++++++++++++++++ .../production/promotion-config/ida-ndwi.json | 84 +++++++++++++++++++ .../promotion-config/marsh-difference.json | 83 ++++++++++++++++++ .../promotion-config/marsh-ida.json | 84 +++++++++++++++++++ .../dataset-config/ida-ndvi-difference.json | 81 ++++++++++++++++++ .../staging/dataset-config/ida-ndvi.json | 83 ++++++++++++++++++ .../dataset-config/ida-ndwi-difference.json | 81 ++++++++++++++++++ .../staging/dataset-config/ida-ndwi.json | 83 ++++++++++++++++++ .../dataset-config/marsh-difference.json | 82 ++++++++++++++++++ .../staging/dataset-config/marsh-ida.json | 83 ++++++++++++++++++ 12 files changed, 992 insertions(+) create mode 100644 ingestion-data/production/promotion-config/ida-ndvi-difference.json create mode 100644 ingestion-data/production/promotion-config/ida-ndvi.json create mode 100644 ingestion-data/production/promotion-config/ida-ndwi-difference.json create mode 100644 ingestion-data/production/promotion-config/ida-ndwi.json create mode 100644 ingestion-data/production/promotion-config/marsh-difference.json create mode 100644 ingestion-data/production/promotion-config/marsh-ida.json create mode 100644 ingestion-data/staging/dataset-config/ida-ndvi-difference.json create mode 100644 ingestion-data/staging/dataset-config/ida-ndvi.json create mode 100644 ingestion-data/staging/dataset-config/ida-ndwi-difference.json create mode 100644 ingestion-data/staging/dataset-config/ida-ndwi.json create mode 100644 ingestion-data/staging/dataset-config/marsh-difference.json create mode 100644 ingestion-data/staging/dataset-config/marsh-ida.json diff --git a/ingestion-data/production/promotion-config/ida-ndvi-difference.json b/ingestion-data/production/promotion-config/ida-ndvi-difference.json new file mode 100644 index 00000000..3ff74b7f --- /dev/null +++ b/ingestion-data/production/promotion-config/ida-ndvi-difference.json @@ -0,0 +1,82 @@ +{ + "collection": "ida-ndvi-difference", + "title": "NDVI Difference for Pre and Post-Hurricane Ida from PlanetScope", + "description": "Normalized Difference Vegetation Index Difference of before and after Hurricane Ida in Southern Louisiana.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDVI_Difference_2021-08-23_2021-09-09.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)NDVI_Difference_(.*).tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "rdbu", + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + }, + "transfer": true + } \ No newline at end of file diff --git a/ingestion-data/production/promotion-config/ida-ndvi.json b/ingestion-data/production/promotion-config/ida-ndvi.json new file mode 100644 index 00000000..9bdd4b58 --- /dev/null +++ b/ingestion-data/production/promotion-config/ida-ndvi.json @@ -0,0 +1,84 @@ +{ + "collection": "ida-ndvi", + "title": "NDVI for Pre and Post-Hurricane Ida from PlanetScope", + "description": "Normalized Difference Vegetation Index before and after Hurricane Ida in Southern Louisiana.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDVI_Pre_Hurricane_cog_2021-08-23.tif", + "s3://veda-data-store-staging/planet-indices-v2/NDVI_Post_Hurricane_cog_2021-09-09.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)NDVI_(.*)_Hurricane_cog_(.*).tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "rdylgn", + "nodata": -999, + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + }, + "transfer": true + } \ No newline at end of file diff --git a/ingestion-data/production/promotion-config/ida-ndwi-difference.json b/ingestion-data/production/promotion-config/ida-ndwi-difference.json new file mode 100644 index 00000000..1366aa4a --- /dev/null +++ b/ingestion-data/production/promotion-config/ida-ndwi-difference.json @@ -0,0 +1,82 @@ +{ + "collection": "ida-ndwi-difference", + "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope", + "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)NDWI_Difference_(.*).tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "rdbu", + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + }, + "transfer": true + } \ No newline at end of file diff --git a/ingestion-data/production/promotion-config/ida-ndwi.json b/ingestion-data/production/promotion-config/ida-ndwi.json new file mode 100644 index 00000000..40e6f4a4 --- /dev/null +++ b/ingestion-data/production/promotion-config/ida-ndwi.json @@ -0,0 +1,84 @@ +{ + "collection": "ida-ndwi", + "title": "NDWI for Pre and Post-Hurricane Ida from PlanetScope", + "description": "Normalized Difference Water Index before and after Hurricane Ida in Southern Louisiana.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Pre_Hurricane_cog_2021-08-23.tif", + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Post_Hurricane_cog_2021-09-09.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)NDWI_(.*)_Hurricane_cog_(.*).tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "rdylbu", + "nodata": -999, + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + }, + "transfer": true + } \ No newline at end of file diff --git a/ingestion-data/production/promotion-config/marsh-difference.json b/ingestion-data/production/promotion-config/marsh-difference.json new file mode 100644 index 00000000..d094427d --- /dev/null +++ b/ingestion-data/production/promotion-config/marsh-difference.json @@ -0,0 +1,83 @@ +{ + "collection": "marsh-difference", + "title": "Salt Marshes Difference for Pre and Post-Hurricane Ida", + "description": "Salt marsh distribution from UNEP-WCMC classified based on NDVI and NDWI.", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/wetlands/difference_pre_post_ida_cog_2021-08-23_2021-09-09.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "wetlands/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)difference_pre_post_ida_cog_2021-08-23_2021-09-09.tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "bwr", + "nodata": 0, + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + }, + "transfer": true + } \ No newline at end of file diff --git a/ingestion-data/production/promotion-config/marsh-ida.json b/ingestion-data/production/promotion-config/marsh-ida.json new file mode 100644 index 00000000..0d24f20d --- /dev/null +++ b/ingestion-data/production/promotion-config/marsh-ida.json @@ -0,0 +1,84 @@ +{ + "collection": "marsh-ida", + "title": "Salt Marsh Classification Pre-Ida (Southern Louisiana)", + "description": "Salt Marsh Classification Pre-Ida (Southern Louisiana)", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/wetlands/post_ida_classified_cog_2021-09-09.tif", + "s3://veda-data-store-staging/wetlands/pre_ida_classified_cog_2021-08-23.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "wetlands/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)_ida_classified_cog_(.*).tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "reds", + "nodata": 0, + "assets": [ + "cog_default" + ], + "rescale": [[0, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + }, + "transfer": true + } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/ida-ndvi-difference.json b/ingestion-data/staging/dataset-config/ida-ndvi-difference.json new file mode 100644 index 00000000..a46600b5 --- /dev/null +++ b/ingestion-data/staging/dataset-config/ida-ndvi-difference.json @@ -0,0 +1,81 @@ +{ + "collection": "ida-ndvi-difference", + "title": "NDVI Difference for Pre and Post-Hurricane Ida from PlanetScope", + "description": "Normalized Difference Vegetation Index Difference of before and after Hurricane Ida in Southern Louisiana.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDVI_Difference_2021-08-23_2021-09-09.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)NDVI_Difference_(.*).tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "rdbu", + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + } + } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/ida-ndvi.json b/ingestion-data/staging/dataset-config/ida-ndvi.json new file mode 100644 index 00000000..db49b360 --- /dev/null +++ b/ingestion-data/staging/dataset-config/ida-ndvi.json @@ -0,0 +1,83 @@ +{ + "collection": "ida-ndvi", + "title": "NDVI for Pre and Post-Hurricane Ida from PlanetScope", + "description": "Normalized Difference Vegetation Index before and after Hurricane Ida in Southern Louisiana.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDVI_Pre_Hurricane_cog_2021-08-23.tif", + "s3://veda-data-store-staging/planet-indices-v2/NDVI_Post_Hurricane_cog_2021-09-09.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)NDVI_(.*)_Hurricane_cog_(.*).tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "rdylgn", + "nodata": -999, + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + } + } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/ida-ndwi-difference.json b/ingestion-data/staging/dataset-config/ida-ndwi-difference.json new file mode 100644 index 00000000..416cb728 --- /dev/null +++ b/ingestion-data/staging/dataset-config/ida-ndwi-difference.json @@ -0,0 +1,81 @@ +{ + "collection": "ida-ndwi-difference", + "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope", + "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)NDWI_Difference_(.*).tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "rdbu", + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + } + } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/ida-ndwi.json b/ingestion-data/staging/dataset-config/ida-ndwi.json new file mode 100644 index 00000000..a75ee1ca --- /dev/null +++ b/ingestion-data/staging/dataset-config/ida-ndwi.json @@ -0,0 +1,83 @@ +{ + "collection": "ida-ndwi", + "title": "NDWI for Pre and Post-Hurricane Ida from PlanetScope", + "description": "Normalized Difference Water Index before and after Hurricane Ida in Southern Louisiana.", + "license": "CC0-1.0", + "is_periodic": true, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Pre_Hurricane_cog_2021-08-23.tif", + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Post_Hurricane_cog_2021-09-09.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)NDWI_(.*)_Hurricane_cog_(.*).tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "rdylbu", + "nodata": -999, + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + } + } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/marsh-difference.json b/ingestion-data/staging/dataset-config/marsh-difference.json new file mode 100644 index 00000000..d1de4167 --- /dev/null +++ b/ingestion-data/staging/dataset-config/marsh-difference.json @@ -0,0 +1,82 @@ +{ + "collection": "marsh-difference", + "title": "Salt Marshes Difference for Pre and Post-Hurricane Ida", + "description": "Salt marsh distribution from UNEP-WCMC classified based on NDVI and NDWI.", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/wetlands/difference_pre_post_ida_cog_2021-08-23_2021-09-09.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "wetlands/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)difference_pre_post_ida_cog_2021-08-23_2021-09-09.tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "bwr", + "nodata": 0, + "assets": [ + "cog_default" + ], + "rescale": [[-1, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + } + } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/marsh-ida.json b/ingestion-data/staging/dataset-config/marsh-ida.json new file mode 100644 index 00000000..77ef5954 --- /dev/null +++ b/ingestion-data/staging/dataset-config/marsh-ida.json @@ -0,0 +1,83 @@ +{ + "collection": "marsh-ida", + "title": "Salt Marsh Classification Pre-Ida (Southern Louisiana)", + "description": "Salt Marsh Classification Pre-Ida (Southern Louisiana)", + "license": "CC0-1.0", + "is_periodic": false, + "time_density": "day", + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00Z", + "enddate": "2021-09-09T23:59:59Z" + }, + "sample_files": [ + "s3://veda-data-store-staging/wetlands/post_ida_classified_cog_2021-09-09.tif", + "s3://veda-data-store-staging/wetlands/pre_ida_classified_cog_2021-08-23.tif" + ], + "discovery_items": [ + { + "discovery": "s3", + "cogify": false, + "upload": false, + "dry_run": false, + "prefix": "wetlands/", + "bucket": "veda-data-store-staging", + "filename_regex": "(.*)_ida_classified_cog_(.*).tif$" + } + ], + "data_type": "cog", + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [1], + "colormap_name": "reds", + "nodata": 0, + "assets": [ + "cog_default" + ], + "rescale": [[0, 1]], + "title": "VEDA Dashboard Render Parameters" + } + }, + "assets": { + "thumbnail": { + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "title": "Thumbnail", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + } + } \ No newline at end of file From 431c8e409e535430041118f5ea562819f25f996e Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Fri, 14 Feb 2025 12:35:50 -0800 Subject: [PATCH 03/13] feat: update to use sm2a for staging promotion (#297) --- .github/workflows/pr.yml | 48 +++------- .github/workflows/promote.yml | 2 +- .../dataset-config/test-sm2a-staging.json | 90 +++++++++++++++++++ .../staging/dataset-config/test.json | 0 ...to_production.py => promote_collection.py} | 60 +++++++++++-- 5 files changed, 153 insertions(+), 47 deletions(-) create mode 100644 ingestion-data/staging/dataset-config/test-sm2a-staging.json delete mode 100644 ingestion-data/staging/dataset-config/test.json rename scripts/{promote_to_production.py => promote_collection.py} (58%) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ff39dd7c..5d1607b6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -84,23 +84,6 @@ jobs: done echo "added_files_count: ${ADDED_FILES_COUNT}" - # Uses service client creds to get token - # No username/password needed - - name: Get auth token - id: get-token - run: | - echo "Vars: $vars" - response=$(curl -X POST \ - ${{ vars.STAGING_COGNITO_DOMAIN }}/oauth2/token \ - -H "Content-Type: application/x-www-form-urlencoded" \ - -d "grant_type=client_credentials" \ - -d "client_id=${{ vars.STAGING_CLIENT_ID }}" \ - -d "client_secret=${{ secrets.STAGING_CLIENT_SECRET }}" - ) - - access_token=$(echo "$response" | jq -r '.access_token') - echo "ACCESS_TOKEN=$access_token" >> $GITHUB_OUTPUT - # Makes request to /dataset/publish endpoint # Outputs only files that were successfully published # Used by other steps @@ -110,23 +93,14 @@ jobs: id: publish-collections env: ADDED_FILES: ${{ steps.changed-files.outputs.added_files }} - WORKFLOWS_URL: ${{ vars.STAGING_WORKFLOWS_URL }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - AUTH_TOKEN: ${{ steps.get-token.outputs.ACCESS_TOKEN }} COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }} + STAGING_SM2A_ADMIN_USERNAME: ${{ secrets.STAGING_SM2A_ADMIN_USERNAME }} + STAGING_SM2A_ADMIN_PASSWORD: ${{ secrets.STAGING_SM2A_ADMIN_PASSWORD }} + STAGING_SM2A_API_URL: ${{ vars.STAGING_SM2A_API_URL }} + DATASET_DAG_NAME: ${{ vars.DATASET_DAG_NAME }} run: | - if [ -z "$WORKFLOWS_URL" ]; then - echo "WORKFLOWS_URL is not set" - exit 1 - fi - - if [ -z "$AUTH_TOKEN" ]; then - echo "AUTH_TOKEN is not set" - exit 1 - fi - - publish_url="${WORKFLOWS_URL%/}/dataset/publish" - bearer_token=$AUTH_TOKEN + pip install -r ./scripts/requirements.txt # Track successful publications all_failed=true @@ -141,16 +115,14 @@ jobs: collection_id=$(jq -r '.collection' "$file") echo "Publishing $collection_id" - response=$(curl -s -w "%{http_code}" -o response.txt -X POST "$publish_url" \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $AUTH_TOKEN" \ - -d "$dataset_config" - ) - status_code=$(tail -n1 <<< "$response") + response=$(python3 ./scripts/promote_collection.py "$file" "staging") + echo "Processed file: $file" + status_code=$(echo "$response" | jq -r '.statusCode') + echo "Status Code: $status_code" # Update status message based on response code - if [ "$status_code" -eq 200 ] || [ "$status_code" -eq 201 ]; then + if [ $status_code -eq 200 ] || [ $status_code -eq 201 ]; then echo "$collection_id successfully published ✅" status_message+="- **$collection_id**: Successfully published ✅ " diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 3fe1f8fa..e60cacfc 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -69,6 +69,6 @@ jobs: run: | pip install -r ./scripts/requirements.txt for file in downloaded-files/*.json; do - python3 ./scripts/promote_to_production.py "$file" + python3 ./scripts/promote_collection.py "$file" "production" echo "Processed file: $file" done diff --git a/ingestion-data/staging/dataset-config/test-sm2a-staging.json b/ingestion-data/staging/dataset-config/test-sm2a-staging.json new file mode 100644 index 00000000..1aefa8a6 --- /dev/null +++ b/ingestion-data/staging/dataset-config/test-sm2a-staging.json @@ -0,0 +1,90 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00.000Z", + "enddate": "2021-09-09T23:59:59.000Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)NDWI_Difference_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + }, + "collection": "ida-ndwi-difference-TEST-SM2A-STAGING", + "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-SM2A-STAGING", + "dashboard:time_density": "day", + "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.", + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [ + 1 + ], + "colormap_name": "rdbu", + "assets": [ + "cog_default" + ], + "rescale": [ + [ + -1, + 1 + ] + ], + "title": "VEDA Dashboard Render Parameters" + } + }, + "dashboard:is_periodic": true + } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/test.json b/ingestion-data/staging/dataset-config/test.json deleted file mode 100644 index e69de29b..00000000 diff --git a/scripts/promote_to_production.py b/scripts/promote_collection.py similarity index 58% rename from scripts/promote_to_production.py rename to scripts/promote_collection.py index 9b618966..b4ab15d3 100644 --- a/scripts/promote_to_production.py +++ b/scripts/promote_collection.py @@ -32,6 +32,46 @@ def validate_discovery_item_config(item: Dict[str, Any]) -> Dict[str, Any]: return item +def publish_to_staging(payload): + base_api_url = os.getenv("STAGING_SM2A_API_URL") + dataset_pipeline_dag = os.getenv("DATASET_DAG_NAME", "veda_dataset_pipeline") + username = os.getenv("STAGING_SM2A_ADMIN_USERNAME") + password = os.getenv("STAGING_SM2A_ADMIN_PASSWORD") + + api_token = b64encode(f"{username}:{password}".encode()).decode() + + if not base_api_url or not api_token: + raise ValueError( + "STAGING_SM2A_API_URL or STAGING_SM2A_ADMIN_USERNAME" + + " or STAGING_SM2A_ADMIN_PASSWORD is not" + + " set in the environment variables." + ) + + headers = { + "Content-Type": "application/json", + "Authorization": "Basic " + api_token, + } + + body = { + **payload, + "dag_run_id": f"{dataset_pipeline_dag}-{uuid.uuid4()}", + "note": "Run from GitHub Actions veda-data", + } + http_conn = http.client.HTTPSConnection(base_api_url) + http_conn.request( + "POST", + f"/api/v1/dags/{dataset_pipeline_dag}/dagRuns", + json.dumps(body), + headers, + ) + response = http_conn.getresponse() + response_data = response.read() + http_conn.close() + + print(json.dumps({"statusCode": response.status})) + return {"statusCode": response.status, "body": response_data.decode()} + + def promote_to_production(payload): base_api_url = os.getenv("SM2A_API_URL") promotion_dag = os.getenv("PROMOTION_DAG_NAME", "veda_promotion_pipeline") @@ -39,8 +79,6 @@ def promote_to_production(payload): password = os.getenv("SM2A_ADMIN_PASSWORD") api_token = b64encode(f"{username}:{password}".encode()).decode() - print(password) - print(api_token) if not base_api_url or not api_token: raise ValueError( @@ -53,18 +91,18 @@ def promote_to_production(payload): "Authorization": "Basic " + api_token, } + payload["transfer"] = True body = { **payload, "dag_run_id": f"{promotion_dag}-{uuid.uuid4()}", "note": "Run from GitHub Actions veda-data", } http_conn = http.client.HTTPSConnection(base_api_url) - response = http_conn.request( + http_conn.request( "POST", f"/api/v1/dags/{promotion_dag}/dagRuns", json.dumps(body), headers ) response = http_conn.getresponse() response_data = response.read() - print(f"Response: ${response_data}") http_conn.close() return {"statusCode": response.status, "body": response_data.decode()} @@ -74,14 +112,20 @@ def promote_to_production(payload): try: with open(sys.argv[1], "r") as file: input = json.load(file) - input["transfer"] = True + stage = sys.argv[2] discovery_items = input.get("discovery_items") validated_discovery_items = [ validate_discovery_item_config(item) for item in discovery_items ] - dag_payload = {"conf": input} - promote_to_production(dag_payload) - + if stage == "production": + promote_to_production(dag_payload) + elif stage == "staging": + publish_to_staging(dag_payload) + + except IndexError: + print("Usage: promote_collection.py ") + except FileNotFoundError: + print(f"Error: File '{sys.argv[1]}' not found.") except json.JSONDecodeError: raise ValueError(f"Invalid JSON content in file {sys.argv[1]}") From 9876efc20843311799e2cb2fa51078f8737fc133 Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Tue, 18 Feb 2025 10:26:41 -0800 Subject: [PATCH 04/13] fix: update promotion workflow to output promotion status (#299) --- .github/workflows/pr.yml | 36 ++++++-- .github/workflows/promote.yml | 10 ++- .../dataset-config/test-changed-files.json | 90 ------------------- .../dataset-config/test-sm2a-staging.json | 90 ------------------- .../{test-changed-files-2.json => test.json} | 4 +- scripts/promote_collection.py | 5 +- 6 files changed, 45 insertions(+), 190 deletions(-) delete mode 100644 ingestion-data/staging/dataset-config/test-changed-files.json rename ingestion-data/staging/dataset-config/{test-changed-files-2.json => test.json} (96%) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5d1607b6..c60a84d0 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -29,8 +29,9 @@ jobs: runs-on: ubuntu-latest environment: staging outputs: - publishedCollections: ${{ steps.publish-collections.outputs.success_collections }} - commentId: ${{ steps.init-comment.outputs.COMMENT_ID }} + published_collections: ${{ steps.publish-collections.outputs.success_collections }} + comment_id: ${{ steps.init-comment.outputs.COMMENT_ID }} + has_new_files_to_promote: ${{ steps.changed-files.outputs.added_files_count != 0 }} steps: - uses: actions/checkout@v4 @@ -84,12 +85,19 @@ jobs: done echo "added_files_count: ${ADDED_FILES_COUNT}" + - name: Exit early if no new files + if: ${{ steps.changed-files.outputs.added_files_count == 0 }} + run: | + echo "🕵️ No new files found. Exiting workflow early." + exit 0 + # Makes request to /dataset/publish endpoint # Outputs only files that were successfully published # Used by other steps # If none of the requests are successful, workflow fails # Updates the PR comment with status of collection publication - name: Publish all newly added collections to staging + if: ${{ steps.changed-files.outputs.added_files_count != 0 }} id: publish-collections env: ADDED_FILES: ${{ steps.changed-files.outputs.added_files }} @@ -178,7 +186,23 @@ jobs: ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**" gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY" + # If the workflow fails at any point, the PR comment will be updated + + - name: Update PR comment if no new files + if: ${{ steps.changed-files.outputs.added_files_count == 0 }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }} + run: | + WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body') + UPDATED_BODY="$CURRENT_BODY + + 🕵️ No new files found. No files will be published to staging or promoted to production." + gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY" + create-mdx-files-and-open-pr: + if: ${{ needs.publish-new-datasets.outputs.has_new_files_to_promote == 'true' }} runs-on: ubuntu-latest environment: staging needs: publish-new-datasets @@ -188,12 +212,12 @@ jobs: - name: Use output from publish-new-datasets run: | - echo "The output from the previous step is: ${{ needs.publish-new-datasets.outputs.publishedCollections }}" + echo "The output from the previous step is: ${{ needs.publish-new-datasets.outputs.published_collections }}" # Creates a slim dataset mdx file for each collection based on the dataset config json - name: Create dataset mdx for given collections env: - PUBLISHED_COLLECTION_FILES: ${{ needs.publish-new-datasets.outputs.publishedCollections }} + PUBLISHED_COLLECTION_FILES: ${{ needs.publish-new-datasets.outputs.published_collections }} run: | echo $PUBLISHED_COLLECTION_FILES collection_ids="" @@ -212,7 +236,7 @@ jobs: - name: Prepare temp directory for artifacts run: | mkdir -p /tmp/upload-artifacts - for file in $(echo "${{ needs.publish-new-datasets.outputs.publishedCollections }}" | tr ',' ' '); do + for file in $(echo "${{ needs.publish-new-datasets.outputs.published_collections }}" | tr ',' ' '); do if [ -f "$file" ]; then cp "$file" /tmp/upload-artifacts/ else @@ -341,7 +365,7 @@ jobs: - name: Create veda-config PR with changes id: create-pr env: - COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }} + COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment_id }} PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }} run: | cd veda-config diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index e60cacfc..647fc8af 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -69,6 +69,14 @@ jobs: run: | pip install -r ./scripts/requirements.txt for file in downloaded-files/*.json; do - python3 ./scripts/promote_collection.py "$file" "production" + collection_id=$(jq -r '.collection' "$file") + response=$(python3 ./scripts/promote_collection.py "$file" "production") echo "Processed file: $file" + status_code=$(echo "$response" | jq -r '.statusCode') + echo "Status Code: $status_code" + if [ $status_code -eq 200 ] || [ $status_code -eq 201 ]; then + echo "$collection_id successfully promoted to Production ✅" + else + echo "$collection_id failed to promote to Production ❌" + fi done diff --git a/ingestion-data/staging/dataset-config/test-changed-files.json b/ingestion-data/staging/dataset-config/test-changed-files.json deleted file mode 100644 index af614ab1..00000000 --- a/ingestion-data/staging/dataset-config/test-changed-files.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "license": "CC0-1.0", - "stac_version": "1.0.0", - "links": [], - "spatial_extent": { - "xmin": -90.85, - "ymin": 29.03, - "xmax": -90.06, - "ymax": 29.65 - }, - "temporal_extent": { - "startdate": "2021-08-23T00:00:00.000Z", - "enddate": "2021-09-09T23:59:59.000Z" - }, - "discovery_items": [ - { - "upload": false, - "cogify": false, - "dry_run": false, - "filename_regex": "(.*)NDWI_Difference_(.*).tif$", - "use_multithreading": false, - "discovery": "s3", - "prefix": "planet-indices-v2/", - "bucket": "veda-data-store-staging" - } - ], - "sample_files": [ - "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif" - ], - "data_type": "cog", - "stac_extensions": [ - "https://stac-extensions.github.io/render/v1.0.0/schema.json", - "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" - ], - "item_assets": { - "cog_default": { - "type": "image/tiff; application=geotiff; profile=cloud-optimized", - "roles": [ - "data", - "layer" - ], - "title": "Default COG Layer", - "description": "Cloud optimized default layer to display on map" - } - }, - "providers": [ - { - "name": "NASA VEDA", - "roles": [ - "host" - ], - "url": "https://www.earthdata.nasa.gov/dashboard/" - } - ], - "assets": { - "thumbnail": { - "title": "Thumbnail", - "type": "image/jpeg", - "roles": [ - "thumbnail" - ], - "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", - "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" - } - }, - "collection": "ida-ndwi-difference-TEST", - "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST", - "dashboard:time_density": "day", - "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.", - "renders": { - "dashboard": { - "resampling": "nearest", - "bidx": [ - 1 - ], - "colormap_name": "rdbu", - "assets": [ - "cog_default" - ], - "rescale": [ - [ - -1, - 1 - ] - ], - "title": "VEDA Dashboard Render Parameters" - } - }, - "dashboard:is_periodic": true - } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/test-sm2a-staging.json b/ingestion-data/staging/dataset-config/test-sm2a-staging.json index 1aefa8a6..e69de29b 100644 --- a/ingestion-data/staging/dataset-config/test-sm2a-staging.json +++ b/ingestion-data/staging/dataset-config/test-sm2a-staging.json @@ -1,90 +0,0 @@ -{ - "license": "CC0-1.0", - "stac_version": "1.0.0", - "links": [], - "spatial_extent": { - "xmin": -90.85, - "ymin": 29.03, - "xmax": -90.06, - "ymax": 29.65 - }, - "temporal_extent": { - "startdate": "2021-08-23T00:00:00.000Z", - "enddate": "2021-09-09T23:59:59.000Z" - }, - "discovery_items": [ - { - "upload": false, - "cogify": false, - "dry_run": false, - "filename_regex": "(.*)NDWI_Difference_(.*).tif$", - "use_multithreading": false, - "discovery": "s3", - "prefix": "planet-indices-v2/", - "bucket": "veda-data-store-staging" - } - ], - "sample_files": [ - "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif" - ], - "data_type": "cog", - "stac_extensions": [ - "https://stac-extensions.github.io/render/v1.0.0/schema.json", - "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" - ], - "item_assets": { - "cog_default": { - "type": "image/tiff; application=geotiff; profile=cloud-optimized", - "roles": [ - "data", - "layer" - ], - "title": "Default COG Layer", - "description": "Cloud optimized default layer to display on map" - } - }, - "providers": [ - { - "name": "NASA VEDA", - "roles": [ - "host" - ], - "url": "https://www.earthdata.nasa.gov/dashboard/" - } - ], - "assets": { - "thumbnail": { - "title": "Thumbnail", - "type": "image/jpeg", - "roles": [ - "thumbnail" - ], - "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", - "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" - } - }, - "collection": "ida-ndwi-difference-TEST-SM2A-STAGING", - "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-SM2A-STAGING", - "dashboard:time_density": "day", - "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.", - "renders": { - "dashboard": { - "resampling": "nearest", - "bidx": [ - 1 - ], - "colormap_name": "rdbu", - "assets": [ - "cog_default" - ], - "rescale": [ - [ - -1, - 1 - ] - ], - "title": "VEDA Dashboard Render Parameters" - } - }, - "dashboard:is_periodic": true - } \ No newline at end of file diff --git a/ingestion-data/staging/dataset-config/test-changed-files-2.json b/ingestion-data/staging/dataset-config/test.json similarity index 96% rename from ingestion-data/staging/dataset-config/test-changed-files-2.json rename to ingestion-data/staging/dataset-config/test.json index 22e33065..7e2a0dc5 100644 --- a/ingestion-data/staging/dataset-config/test-changed-files-2.json +++ b/ingestion-data/staging/dataset-config/test.json @@ -63,8 +63,8 @@ "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" } }, - "collection": "ida-ndwi-difference-TEST-2", - "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-2", + "collection": "ida-ndwi-difference-TEST-SM2A-PRODUCTION", + "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-SM2A-PRODUCTION", "dashboard:time_density": "day", "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.", "renders": { diff --git a/scripts/promote_collection.py b/scripts/promote_collection.py index b4ab15d3..e2ea38fd 100644 --- a/scripts/promote_collection.py +++ b/scripts/promote_collection.py @@ -69,6 +69,7 @@ def publish_to_staging(payload): http_conn.close() print(json.dumps({"statusCode": response.status})) + print(response_data.decode()) return {"statusCode": response.status, "body": response_data.decode()} @@ -91,7 +92,7 @@ def promote_to_production(payload): "Authorization": "Basic " + api_token, } - payload["transfer"] = True + payload["conf"]["transfer"] = True body = { **payload, "dag_run_id": f"{promotion_dag}-{uuid.uuid4()}", @@ -105,6 +106,8 @@ def promote_to_production(payload): response_data = response.read() http_conn.close() + print(json.dumps({"statusCode": response.status})) + print(response_data.decode()) return {"statusCode": response.status, "body": response_data.decode()} From 7240c5a70dfa9f00abab591dbb3bd0b6b3011635 Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Tue, 18 Feb 2025 14:27:55 -0800 Subject: [PATCH 05/13] fix: fix pr.yml issues and modify end quotes (#301) --- .github/workflows/pr.yml | 19 ++-- .../dataset-config/test-debugging.json | 90 +++++++++++++++++++ 2 files changed, 98 insertions(+), 11 deletions(-) create mode 100644 ingestion-data/staging/dataset-config/test-debugging.json diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c60a84d0..37cea549 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -113,8 +113,7 @@ jobs: # Track successful publications all_failed=true declare -a success_collections=() - status_message='### Collection Publication Status - ' + status_message="### Collection Publication Status" for file in ${ADDED_FILES}; do echo $file @@ -126,20 +125,18 @@ jobs: response=$(python3 ./scripts/promote_collection.py "$file" "staging") echo "Processed file: $file" - status_code=$(echo "$response" | jq -r '.statusCode') + status_code=$(echo "$response" | jq -r '.statusCode' | head -n1) echo "Status Code: $status_code" # Update status message based on response code - if [ $status_code -eq 200 ] || [ $status_code -eq 201 ]; then + if [[ $status_code -eq 200 ]] || [[ $status_code -eq 201 ]]; then echo "$collection_id successfully published ✅" - status_message+="- **$collection_id**: Successfully published ✅ - " + status_message+="- **$collection_id**: Successfully published ✅" success_collections+=("$file") all_failed=false else echo "$collection_id failed to publish ❌" - status_message+="- **$collection_id**: Failed to publish. Error code $status_code. ❌ - " + status_message+="- **$collection_id**: Failed to publish. Error code $status_code. ❌" fi else echo "File $file does not exist" @@ -409,7 +406,7 @@ jobs: if: success() env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }} + COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment_id }} run: | PR_URL=${{ steps.create-pr.outputs.PR_URL }} CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body') @@ -422,7 +419,7 @@ jobs: if: failure() && steps.create-pr.outcome == 'failure' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }} + COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment_id }} run: | CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body') UPDATED_BODY="$CURRENT_BODY @@ -435,7 +432,7 @@ jobs: if: failure() && steps.create-pr.outcome != 'failure' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }} + COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment_id }} run: | WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body') diff --git a/ingestion-data/staging/dataset-config/test-debugging.json b/ingestion-data/staging/dataset-config/test-debugging.json new file mode 100644 index 00000000..b525600b --- /dev/null +++ b/ingestion-data/staging/dataset-config/test-debugging.json @@ -0,0 +1,90 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -90.85, + "ymin": 29.03, + "xmax": -90.06, + "ymax": 29.65 + }, + "temporal_extent": { + "startdate": "2021-08-23T00:00:00.000Z", + "enddate": "2021-09-09T23:59:59.000Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)NDWI_Difference_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "planet-indices-v2/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/jpeg", + "roles": [ + "thumbnail" + ], + "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg", + "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)" + } + }, + "collection": "ida-ndwi-difference-TEST-DEBUG", + "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-DEBUG", + "dashboard:time_density": "day", + "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.", + "renders": { + "dashboard": { + "resampling": "nearest", + "bidx": [ + 1 + ], + "colormap_name": "rdbu", + "assets": [ + "cog_default" + ], + "rescale": [ + [ + -1, + 1 + ] + ], + "title": "VEDA Dashboard Render Parameters" + } + }, + "dashboard:is_periodic": true + } \ No newline at end of file From 48a4e08c0c1efc1dc79343ae141d448e0cf7221b Mon Sep 17 00:00:00 2001 From: "veda-github-actor[bot]" <192275278+veda-github-actor[bot]@users.noreply.github.com> Date: Wed, 19 Feb 2025 10:45:23 -0700 Subject: [PATCH 06/13] Ingest Request for blizzard-millera (#277) * Create ingestion-data/staging/dataset-config/blizzard-millera.json * use custom rgb colormap --------- Co-authored-by: veda-github-actor[bot] <192275278+veda-github-actor[bot]@users.noreply.github.com> Co-authored-by: Alexandra Kirk --- .../dataset-config/blizzard-millera.json | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 ingestion-data/staging/dataset-config/blizzard-millera.json diff --git a/ingestion-data/staging/dataset-config/blizzard-millera.json b/ingestion-data/staging/dataset-config/blizzard-millera.json new file mode 100644 index 00000000..d91f0942 --- /dev/null +++ b/ingestion-data/staging/dataset-config/blizzard-millera.json @@ -0,0 +1,90 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -83.5, + "ymin": 34.1, + "xmax": -64.8, + "ymax": 48.9 + }, + "temporal_extent": { + "startdate": "1950-01-01T00:00:00Z", + "enddate": "1950-01-01T23:59:59Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)MillerA_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "Blizzards/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/Blizzards/MillerA_1950-01-01.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/png", + "roles": [ + "thumbnail" + ], + "description": "Photo by Andrew Blackford (Near-blizzard conditions in Maria Stein, Ohio on December 23, 2022)", + "href": "https://thumbnails.openveda.cloud/blizzards-cover.png" + } + }, + "collection": "blizzard-millera", + "title": "Miller A Snowfall Footprint", + "description": "Single value raster displaying the general snowfall footprint of a Miller A type of mid-latitude cyclone.", + "is_periodic": true, + "time_density": "day", + "datetime_range": "day", + "renders": { + "dashboard": { + "colormap": { + "0": [ + 22, + 158, + 242, + 255 + ] + }, + "assets": [ + "cog_default" + ], + "title": "VEDA Dashboard Render Parameters" + } + }, + "transfer": true, + "dashboard:time_density": "day" +} \ No newline at end of file From 4bf6fdb50eb4096ff5ca685f513e023c5eafe624 Mon Sep 17 00:00:00 2001 From: "veda-github-actor[bot]" <192275278+veda-github-actor[bot]@users.noreply.github.com> Date: Wed, 19 Feb 2025 11:14:52 -0700 Subject: [PATCH 07/13] Ingest Request for blizzard-millerb (#278) * Create ingestion-data/staging/dataset-config/blizzard-millerb.json * use custom rgb colormap --------- Co-authored-by: veda-github-actor[bot] <192275278+veda-github-actor[bot]@users.noreply.github.com> Co-authored-by: Alexandra Kirk --- .../dataset-config/blizzard-millerb.json | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 ingestion-data/staging/dataset-config/blizzard-millerb.json diff --git a/ingestion-data/staging/dataset-config/blizzard-millerb.json b/ingestion-data/staging/dataset-config/blizzard-millerb.json new file mode 100644 index 00000000..8f0612c6 --- /dev/null +++ b/ingestion-data/staging/dataset-config/blizzard-millerb.json @@ -0,0 +1,90 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -98, + "ymin": 37.5, + "xmax": -64.8, + "ymax": 48.9 + }, + "temporal_extent": { + "startdate": "1950-01-01T00:00:00Z", + "enddate": "1950-01-01T23:59:59Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)MillerB_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "Blizzards/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/Blizzards/MillerB_1950-01-01.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/png", + "roles": [ + "thumbnail" + ], + "description": "Photo by Andrew Blackford (Near-blizzard conditions in Maria Stein, Ohio on December 23, 2022)", + "href": "https://thumbnails.openveda.cloud/blizzards-cover.png" + } + }, + "collection": "blizzard-millerb", + "title": "Miller B Snowfall Footprint", + "description": "Single value raster displaying the general snowfall footprint of a Miller B type of mid-latitude cyclone.", + "is_periodic": true, + "time_density": "day", + "datetime_range": "day", + "renders": { + "dashboard": { + "colormap": { + "0": [ + 22, + 158, + 242, + 255 + ] + }, + "assets": [ + "cog_default" + ], + "title": "VEDA Dashboard Render Parameters" + } + }, + "transfer": true, + "dashboard:time_density": "day" +} \ No newline at end of file From 717a9bff40b7248d8ec8d6cf22884bd70f556675 Mon Sep 17 00:00:00 2001 From: "veda-github-actor[bot]" <192275278+veda-github-actor[bot]@users.noreply.github.com> Date: Wed, 19 Feb 2025 11:21:21 -0700 Subject: [PATCH 08/13] Ingest Request for blizzard-alley (#279) * Create ingestion-data/staging/dataset-config/blizzard-alley.json * update via UI --------- Co-authored-by: veda-github-actor[bot] <192275278+veda-github-actor[bot]@users.noreply.github.com> --- .../dataset-config/blizzard-alley.json | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 ingestion-data/staging/dataset-config/blizzard-alley.json diff --git a/ingestion-data/staging/dataset-config/blizzard-alley.json b/ingestion-data/staging/dataset-config/blizzard-alley.json new file mode 100644 index 00000000..605219b6 --- /dev/null +++ b/ingestion-data/staging/dataset-config/blizzard-alley.json @@ -0,0 +1,90 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -105.3, + "ymin": 36.8, + "xmax": -89.4, + "ymax": 49.8 + }, + "temporal_extent": { + "startdate": "1950-01-01T00:00:00Z", + "enddate": "1950-01-01T23:59:59Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)BlizzardAlley_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "Blizzards/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/Blizzards/BlizzardAlley_1950-01-01.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/png", + "roles": [ + "thumbnail" + ], + "description": "Photo by Andrew Blackford (Near-blizzard conditions in Maria Stein, Ohio on December 23, 2022)", + "href": "https://thumbnails.openveda.cloud/blizzards-cover.png" + } + }, + "collection": "blizzard-alley", + "title": "Blizzard Alley", + "description": "Single value raster displaying the region of the USA known as Blizzard Alley, where at least 10-20 blizzards have occurred since 1950. Eastern North Dakota seen 50-100+ in the same time period.", + "is_periodic": true, + "time_density": "day", + "datetime_range": "day", + "renders": { + "dashboard": { + "colormap": { + "0": [ + 22, + 158, + 242, + 255 + ] + }, + "assets": [ + "cog_default" + ], + "title": "VEDA Dashboard Render Parameters" + } + }, + "transfer": true, + "dashboard:time_density": "day" +} \ No newline at end of file From d384b3dfb231008911e76589ceda56579adf95bf Mon Sep 17 00:00:00 2001 From: "veda-github-actor[bot]" <192275278+veda-github-actor[bot]@users.noreply.github.com> Date: Wed, 19 Feb 2025 15:48:21 -0700 Subject: [PATCH 09/13] Ingest Request for blizzard-clipper (#280) --- .../dataset-config/blizzard-clipper.json | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 ingestion-data/staging/dataset-config/blizzard-clipper.json diff --git a/ingestion-data/staging/dataset-config/blizzard-clipper.json b/ingestion-data/staging/dataset-config/blizzard-clipper.json new file mode 100644 index 00000000..d455ac70 --- /dev/null +++ b/ingestion-data/staging/dataset-config/blizzard-clipper.json @@ -0,0 +1,91 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -119.4, + "ymin": 37.5, + "xmax": -79.5, + "ymax": 59.8 + }, + "temporal_extent": { + "startdate": "1950-01-01T00:00:00Z", + "enddate": "1950-01-01T23:59:59Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)AB_SK_MB_Clipper_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "Blizzards/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/Blizzards/AB_SK_MB_Clipper_1950-01-01.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/png", + "roles": [ + "thumbnail" + ], + "description": "Photo by Andrew Blackford (Near-blizzard conditions in Maria Stein, Ohio on December 23, 2022)", + "href": "https://thumbnails.openveda.cloud/blizzards-cover.png" + } + }, + "collection": "blizzard-clipper", + "title": "AB/SK/MB Clipper Snowfall Footprint", + "description": "Single value raster displaying the general snowfall footprint of Alberta Clipper, Saskatchewan Screamer, and Manitoba Mauler types of mid-latitude cyclones.", + "is_periodic": false, + "time_density": "day", + "datetime_range": "day", + "renders": { + "dashboard": { + "colormap": { + "0": [ + 22, + 158, + 242, + 255 + ] + }, + "assets": [ + "cog_default" + ], + "title": "VEDA Dashboard Render Parameters" + } + }, + "transfer": true, + "dashboard:time_density": "day", + "dashboard:is_periodic": false +} \ No newline at end of file From 13986b4cd25f006890ae6959732b8cfa5dfd46fa Mon Sep 17 00:00:00 2001 From: "veda-github-actor[bot]" <192275278+veda-github-actor[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 07:27:26 -0700 Subject: [PATCH 10/13] Ingest Request for blizzard-co-low (#281) --- .../dataset-config/blizzard-co-low.json | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 ingestion-data/staging/dataset-config/blizzard-co-low.json diff --git a/ingestion-data/staging/dataset-config/blizzard-co-low.json b/ingestion-data/staging/dataset-config/blizzard-co-low.json new file mode 100644 index 00000000..dddf6e36 --- /dev/null +++ b/ingestion-data/staging/dataset-config/blizzard-co-low.json @@ -0,0 +1,90 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -110, + "ymin": 38.3, + "xmax": -80.9, + "ymax": 48.9 + }, + "temporal_extent": { + "startdate": "1950-01-01T00:00:00Z", + "enddate": "1950-01-01T23:59:59Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)CO_Low_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "Blizzards/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/Blizzards/CO_Low_1950-01-01.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/png", + "roles": [ + "thumbnail" + ], + "description": "Photo by Andrew Blackford (Near-blizzard conditions in Maria Stein, Ohio on December 23, 2022)", + "href": "https://thumbnails.openveda.cloud/blizzards-cover.png" + } + }, + "collection": "blizzard-co-low", + "title": "Colorado Low Snowfall Footprint", + "description": "Single value raster displaying the general snowfall footprint of a Colorado Low type of mid-latitude cyclone.", + "time_density": "day", + "datetime_range": "day", + "renders": { + "dashboard": { + "colormap": { + "0": [ + 22, + 158, + 242, + 255 + ] + }, + "assets": [ + "cog_default" + ], + "title": "VEDA Dashboard Render Parameters" + } + }, + "transfer": true, + "dashboard:time_density": "day", + "dashboard:is_periodic": false +} \ No newline at end of file From 4d3a66d91b99679042d0a48b40a760cd0a05a5d5 Mon Sep 17 00:00:00 2001 From: "veda-github-actor[bot]" <192275278+veda-github-actor[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 07:37:27 -0700 Subject: [PATCH 11/13] Ingest Request for blizzard-noreaster (#282) --- .../dataset-config/blizzard-noreaster.json | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 ingestion-data/staging/dataset-config/blizzard-noreaster.json diff --git a/ingestion-data/staging/dataset-config/blizzard-noreaster.json b/ingestion-data/staging/dataset-config/blizzard-noreaster.json new file mode 100644 index 00000000..fdf70085 --- /dev/null +++ b/ingestion-data/staging/dataset-config/blizzard-noreaster.json @@ -0,0 +1,89 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -77.7, + "ymin": 35.9, + "xmax": -64.8, + "ymax": 48.9 + }, + "temporal_extent": { + "startdate": "1950-01-01T00:00:00Z", + "enddate": "1950-01-01T23:59:59Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)Noreaster_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "Blizzards/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/Blizzards/Noreaster_1950-01-01.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/png", + "roles": [ + "thumbnail" + ], + "description": "Photo by Andrew Blackford (Near-blizzard conditions in Maria Stein, Ohio on December 23, 2022)", + "href": "https://thumbnails.openveda.cloud/blizzards-cover.png" + } + }, + "collection": "blizzard-noreaster", + "title": "Nor’easter Snowfall Footprint", + "description": "Single value raster displaying the general snowfall footprint of a Nor’easter type of mid-latitude cyclone.", + "time_density": "day", + "datetime_range": "day", + "renders": { + "dashboard": { + "colormap": { + "0": [ + 22, + 158, + 242, + 255 + ] + }, + "assets": [ + "cog_default" + ], + "title": "VEDA Dashboard Render Parameters" + } + }, + "dashboard:time_density": "day", + "dashboard:is_periodic": false +} \ No newline at end of file From cc694117cdc7327555250e058e86a0129c46f69a Mon Sep 17 00:00:00 2001 From: "veda-github-actor[bot]" <192275278+veda-github-actor[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 07:44:23 -0700 Subject: [PATCH 12/13] Ingest Request for blizzard-panhandle-hooker (#283) --- .../blizzard-panhandle-hooker.json | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 ingestion-data/staging/dataset-config/blizzard-panhandle-hooker.json diff --git a/ingestion-data/staging/dataset-config/blizzard-panhandle-hooker.json b/ingestion-data/staging/dataset-config/blizzard-panhandle-hooker.json new file mode 100644 index 00000000..e3f3895c --- /dev/null +++ b/ingestion-data/staging/dataset-config/blizzard-panhandle-hooker.json @@ -0,0 +1,89 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -105, + "ymin": 33.7, + "xmax": -81.8, + "ymax": 50 + }, + "temporal_extent": { + "startdate": "1950-01-01T00:00:00Z", + "enddate": "1950-01-01T23:59:59Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)Panhandle_Hooker_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "Blizzards/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/Blizzards/Panhandle_Hooker_1950-01-01.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/png", + "roles": [ + "thumbnail" + ], + "description": "Photo by Andrew Blackford (Near-blizzard conditions in Maria Stein, Ohio on December 23, 2022)", + "href": "https://thumbnails.openveda.cloud/blizzards-cover.png" + } + }, + "collection": "blizzard-panhandle-hooker", + "title": "Panhandle Hooker Snowfall Footprint", + "description": "Single value raster displaying the general snowfall footprint of a Panhandle Hooker type of mid-latitude cyclone.", + "time_density": "day", + "datetime_range": "day", + "renders": { + "dashboard": { + "colormap": { + "0": [ + 22, + 158, + 242, + 255 + ] + }, + "assets": [ + "cog_default" + ], + "title": "VEDA Dashboard Render Parameters" + } + }, + "dashboard:time_density": "day", + "dashboard:is_periodic": false +} \ No newline at end of file From a341392e4de7c85891e34683f40c50e2ce0a901c Mon Sep 17 00:00:00 2001 From: "veda-github-actor[bot]" <192275278+veda-github-actor[bot]@users.noreply.github.com> Date: Thu, 20 Feb 2025 09:28:19 -0700 Subject: [PATCH 13/13] Ingest Request for blizzard-count (#284) * Create ingestion-data/staging/dataset-config/blizzard-count.json * update via UI * update dashboard keys --------- Co-authored-by: veda-github-actor[bot] <192275278+veda-github-actor[bot]@users.noreply.github.com> Co-authored-by: Alexandra Kirk --- .../dataset-config/blizzard-count.json | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 ingestion-data/staging/dataset-config/blizzard-count.json diff --git a/ingestion-data/staging/dataset-config/blizzard-count.json b/ingestion-data/staging/dataset-config/blizzard-count.json new file mode 100644 index 00000000..e259321b --- /dev/null +++ b/ingestion-data/staging/dataset-config/blizzard-count.json @@ -0,0 +1,87 @@ +{ + "license": "CC0-1.0", + "stac_version": "1.0.0", + "links": [], + "spatial_extent": { + "xmin": -110.6, + "ymin": 34.7, + "xmax": -66.7, + "ymax": 49.6 + }, + "temporal_extent": { + "startdate": "2021-12-31T00:00:00Z", + "enddate": "2021-12-31T23:59:59Z" + }, + "discovery_items": [ + { + "upload": false, + "cogify": false, + "dry_run": false, + "filename_regex": "(.*)StormEvents_Blizzards_(.*).tif$", + "use_multithreading": false, + "discovery": "s3", + "prefix": "Blizzards/", + "bucket": "veda-data-store-staging" + } + ], + "sample_files": [ + "s3://veda-data-store-staging/Blizzards/StormEvents_Blizzards_cog_2021-12-31.tif" + ], + "data_type": "cog", + "stac_extensions": [ + "https://stac-extensions.github.io/render/v1.0.0/schema.json", + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" + ], + "item_assets": { + "cog_default": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": [ + "data", + "layer" + ], + "title": "Default COG Layer", + "description": "Cloud optimized default layer to display on map" + } + }, + "providers": [ + { + "name": "NASA VEDA", + "roles": [ + "host" + ], + "url": "https://www.earthdata.nasa.gov/dashboard/" + } + ], + "assets": { + "thumbnail": { + "title": "Thumbnail", + "type": "image/png", + "roles": [ + "thumbnail" + ], + "description": "Photo by Andrew Blackford (Near-blizzard conditions in Maria Stein, Ohio on December 23, 2022)", + "href": "https://thumbnails.openveda.cloud/blizzards-cover.png" + } + }, + "collection": "blizzard-count", + "title": "Blizzard Count 1950-2021", + "description": "Nearest neighbor interpolation of NCEI Storm Events Database confirmed blizzard reports from 1950-2021 in the United States, clipped to the Northern Plains, Midwest, and Northeast.", + "dashboard:is_periodic": true, + "dashboard:time_density": "day", + "datetime_range": "day", + "renders": { + "dashboard": { + "colormap_name": "turbo", + "rescale": [ + [ + 1, + 50 + ] + ], + "assets": [ + "cog_default" + ], + "title": "VEDA Dashboard Render Parameters" + } + } +} \ No newline at end of file