Merge remote-tracking branch 'origin' into feat/blizzard-clipper

.github/workflows/pr.yml

@@ -29,8 +29,9 @@ jobs:
     runs-on: ubuntu-latest
     environment: staging
     outputs:
-      publishedCollections: ${{ steps.publish-collections.outputs.success_collections }}
-      commentId: ${{ steps.init-comment.outputs.COMMENT_ID }}
+      published_collections: ${{ steps.publish-collections.outputs.success_collections }}
+      comment_id: ${{ steps.init-comment.outputs.COMMENT_ID }}
+      has_new_files_to_promote: ${{ steps.changed-files.outputs.added_files_count != 0 }}
     steps:
       - uses: actions/checkout@v4
 
@@ -72,8 +73,7 @@ jobs:
         id: changed-files
         uses: tj-actions/changed-files@v45
         with:
-          files: |
-            **.json
+          files: ingestion-data/staging/dataset-config/**.json
 
       - name: List all newly added files
         env:
@@ -85,55 +85,35 @@ jobs:
           done
           echo "added_files_count: ${ADDED_FILES_COUNT}"
 
-      # Uses service client creds to get token
-      # No username/password needed
-      - name: Get auth token
-        id: get-token
+      - name: Exit early if no new files
+        if: ${{ steps.changed-files.outputs.added_files_count == 0 }}
         run: |
-          echo "Vars: $vars"
-          response=$(curl -X POST \
-            ${{ vars.STAGING_COGNITO_DOMAIN }}/oauth2/token \
-            -H "Content-Type: application/x-www-form-urlencoded" \
-            -d "grant_type=client_credentials" \
-            -d "client_id=${{ vars.STAGING_CLIENT_ID }}" \
-            -d "client_secret=${{ secrets.STAGING_CLIENT_SECRET }}"
-          )
-
-          access_token=$(echo "$response" | jq -r '.access_token')
-          echo "ACCESS_TOKEN=$access_token" >> $GITHUB_OUTPUT
+          echo "🕵️ No new files found. Exiting workflow early."
+          exit 0
 
       # Makes request to /dataset/publish endpoint
       # Outputs only files that were successfully published
       # Used by other steps
       # If none of the requests are successful, workflow fails
       # Updates the PR comment with status of collection publication
       - name: Publish all newly added collections to staging
+        if: ${{ steps.changed-files.outputs.added_files_count != 0 }}
         id: publish-collections
         env:
           ADDED_FILES: ${{ steps.changed-files.outputs.added_files }}
-          WORKFLOWS_URL: ${{ vars.STAGING_WORKFLOWS_URL }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          AUTH_TOKEN: ${{ steps.get-token.outputs.ACCESS_TOKEN }}
           COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+          STAGING_SM2A_ADMIN_USERNAME: ${{ secrets.STAGING_SM2A_ADMIN_USERNAME }}
+          STAGING_SM2A_ADMIN_PASSWORD: ${{ secrets.STAGING_SM2A_ADMIN_PASSWORD }}
+          STAGING_SM2A_API_URL: ${{ vars.STAGING_SM2A_API_URL }}
+          DATASET_DAG_NAME: ${{ vars.DATASET_DAG_NAME }}
         run: |
-          if [ -z "$WORKFLOWS_URL" ]; then
-            echo "WORKFLOWS_URL is not set"
-            exit 1
-          fi
-
-          if [ -z "$AUTH_TOKEN" ]; then
-            echo "AUTH_TOKEN is not set"
-            exit 1
-          fi
-
-          publish_url="${WORKFLOWS_URL%/}/dataset/publish"
-          bearer_token=$AUTH_TOKEN
+          pip install -r ./scripts/requirements.txt
 
           # Track successful publications
           all_failed=true
           declare -a success_collections=()
-          status_message='### Collection Publication Status
-          '
+          status_message="### Collection Publication Status"
 
           for file in ${ADDED_FILES}; do
             echo $file
@@ -142,25 +122,21 @@ jobs:
               collection_id=$(jq -r '.collection' "$file")
 
               echo "Publishing $collection_id"
-              response=$(curl -s -w "%{http_code}" -o response.txt -X POST "$publish_url" \
-                -H "Content-Type: application/json" \
-                -H "Authorization: Bearer $AUTH_TOKEN" \
-                -d "$dataset_config"
-              )
 
-              status_code=$(tail -n1 <<< "$response")
+              response=$(python3 ./scripts/promote_collection.py "$file" "staging")
+              echo "Processed file: $file"
+              status_code=$(echo "$response" | jq -r '.statusCode' | head -n1)
+              echo "Status Code: $status_code"
 
               # Update status message based on response code
-              if [ "$status_code" -eq 200 ] || [ "$status_code" -eq 201 ]; then
+              if [[ $status_code -eq 200 ]] || [[ $status_code -eq 201 ]]; then
                 echo "$collection_id successfully published ✅"
-                status_message+="- **$collection_id**: Successfully published ✅
-                "
+                status_message+="- **$collection_id**: Successfully published ✅"
                 success_collections+=("$file")
                 all_failed=false
               else
                 echo "$collection_id failed to publish ❌"
-                status_message+="- **$collection_id**: Failed to publish. Error code $status_code. ❌
-                "
+                status_message+="- **$collection_id**: Failed to publish. Error code $status_code. ❌"
               fi
             else
               echo "File $file does not exist"
@@ -207,7 +183,23 @@ jobs:
           ** ❌ The workflow run failed. [See logs here]($WORKFLOW_URL)**"
           gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
 
+            # If the workflow fails at any point, the PR comment will be updated
+
+      - name: Update PR comment if no new files
+        if: ${{ steps.changed-files.outputs.added_files_count == 0 }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMMENT_ID: ${{ steps.init-comment.outputs.COMMENT_ID }}
+        run: |
+          WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
+          UPDATED_BODY="$CURRENT_BODY
+
+          🕵️ No new files found. No files will be published to staging or promoted to production."
+          gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
+
   create-mdx-files-and-open-pr:
+    if: ${{ needs.publish-new-datasets.outputs.has_new_files_to_promote == 'true' }}
     runs-on: ubuntu-latest
     environment: staging
     needs: publish-new-datasets
@@ -217,12 +209,12 @@ jobs:
 
       - name: Use output from publish-new-datasets
         run: |
-          echo "The output from the previous step is: ${{ needs.publish-new-datasets.outputs.publishedCollections }}"
+          echo "The output from the previous step is: ${{ needs.publish-new-datasets.outputs.published_collections }}"
 
       # Creates a slim dataset mdx file for each collection based on the dataset config json
       - name: Create dataset mdx for given collections
         env:
-          PUBLISHED_COLLECTION_FILES: ${{ needs.publish-new-datasets.outputs.publishedCollections }}
+          PUBLISHED_COLLECTION_FILES: ${{ needs.publish-new-datasets.outputs.published_collections }}
         run: |
           echo $PUBLISHED_COLLECTION_FILES
           collection_ids=""
@@ -238,14 +230,27 @@ jobs:
           echo "Final collection_ids: $collection_ids"
           echo "collection_ids=${collection_ids}" >> $GITHUB_ENV
 
+      - name: Prepare temp directory for artifacts
+        run: |
+            mkdir -p /tmp/upload-artifacts
+            for file in $(echo "${{ needs.publish-new-datasets.outputs.published_collections }}" | tr ',' ' '); do
+              if [ -f "$file" ]; then
+                cp "$file" /tmp/upload-artifacts/
+              else
+                echo "Warning: File not found - $file"
+              fi
+            done
+
+      - name: List files in temp directory
+        run: ls -lah /tmp/upload-artifacts
+
       - name: Upload published collections files as directory artifact
         uses: actions/upload-artifact@v4
         env:
-          PUBLISHED_COLLECTION_FILES: ${{ needs.publish-new-datasets.outputs.publishedCollections }}
           ARTIFACT_RETENTION_DAYS: ${{ vars.ARTIFACT_RETENTION_DAYS }}
         with:
           name: collections-to-promote-from-${{ github.event.pull_request.number }}
-          path: ${{ env.PUBLISHED_COLLECTION_FILES }}
+          path: /tmp/upload-artifacts/*
           retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }}
 
       - name: Save Workflow Run ID
@@ -357,7 +362,7 @@ jobs:
       - name: Create veda-config PR with changes
         id: create-pr
         env:
-          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment_id }}
           PUBLISHED_COLLECTION_FILES: ${{ steps.publish-collections.outputs.success_collections }}
         run: |
           cd veda-config
@@ -401,7 +406,7 @@ jobs:
         if: success()
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment_id }}
         run: |
          PR_URL=${{ steps.create-pr.outputs.PR_URL }}
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
@@ -414,7 +419,7 @@ jobs:
         if: failure() && steps.create-pr.outcome == 'failure'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment_id }}
         run: |
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')
           UPDATED_BODY="$CURRENT_BODY
@@ -427,7 +432,7 @@ jobs:
         if: failure() && steps.create-pr.outcome != 'failure'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.commentId }}
+          COMMENT_ID: ${{ needs.publish-new-datasets.outputs.comment_id }}
         run: |
           WORKFLOW_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           CURRENT_BODY=$(gh api -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID --jq '.body')

.github/workflows/promote.yml

@@ -69,6 +69,14 @@ jobs:
         run: |
           pip install -r ./scripts/requirements.txt
           for file in downloaded-files/*.json; do
-            python3 ./scripts/promote_to_production.py "$file"
+            collection_id=$(jq -r '.collection' "$file")
+            response=$(python3 ./scripts/promote_collection.py "$file" "production")
             echo "Processed file: $file"
+            status_code=$(echo "$response" | jq -r '.statusCode')
+            echo "Status Code: $status_code"
+             if [ $status_code -eq 200 ] || [ $status_code -eq 201 ]; then
+                echo "$collection_id successfully promoted to Production ✅"
+              else
+                echo "$collection_id failed to promote to Production ❌"
+              fi
           done

ingestion-data/production/dataset-config/test-changed-files.json

@@ -0,0 +1,90 @@
+{
+    "license": "CC0-1.0",
+    "stac_version": "1.0.0",
+    "links": [],
+    "spatial_extent": {
+      "xmin": -90.85,
+      "ymin": 29.03,
+      "xmax": -90.06,
+      "ymax": 29.65
+    },
+    "temporal_extent": {
+      "startdate": "2021-08-23T00:00:00.000Z",
+      "enddate": "2021-09-09T23:59:59.000Z"
+    },
+    "discovery_items": [
+      {
+        "upload": false,
+        "cogify": false,
+        "dry_run": false,
+        "filename_regex": "(.*)NDWI_Difference_(.*).tif$",
+        "use_multithreading": false,
+        "discovery": "s3",
+        "prefix": "planet-indices-v2/",
+        "bucket": "veda-data-store-staging"
+      }
+    ],
+    "sample_files": [
+      "s3://veda-data-store-staging/planet-indices-v2/NDWI_Difference_2021-08-23_2021-09-09.tif"
+    ],
+    "data_type": "cog",
+    "stac_extensions": [
+      "https://stac-extensions.github.io/render/v1.0.0/schema.json",
+      "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json"
+    ],
+    "item_assets": {
+      "cog_default": {
+        "type": "image/tiff; application=geotiff; profile=cloud-optimized",
+        "roles": [
+          "data",
+          "layer"
+        ],
+        "title": "Default COG Layer",
+        "description": "Cloud optimized default layer to display on map"
+      }
+    },
+    "providers": [
+      {
+        "name": "NASA VEDA",
+        "roles": [
+          "host"
+        ],
+        "url": "https://www.earthdata.nasa.gov/dashboard/"
+      }
+    ],
+    "assets": {
+      "thumbnail": {
+        "title": "Thumbnail",
+        "type": "image/jpeg",
+        "roles": [
+          "thumbnail"
+        ],
+        "href": "https://thumbnails.openveda.cloud/louisiana-marsh.jpg",
+        "description": "Photo by [Bridget Besaw](https://www.nature.org/en-us/get-involved/how-to-help/places-we-protect/the-nature-conservancy-in-louisiana-gulf-coast-prairies-and-marshes/) (Wetland landscape across southern Louisiana.)"
+      }
+    },
+    "collection": "ida-ndwi-difference-TEST-SHOULDNT-RUN",
+    "title": "NDWI Difference for Pre and Post-Hurricane Ida from PlanetScope TEST-SHOULDNT-RUN",
+    "dashboard:time_density": "day",
+    "description": "Normalized Difference Water Index Difference of before and after Hurricane Ida in Southern Louisiana.",
+    "renders": {
+      "dashboard": {
+        "resampling": "nearest",
+        "bidx": [
+          1
+        ],
+        "colormap_name": "rdbu",
+        "assets": [
+          "cog_default"
+        ],
+        "rescale": [
+          [
+            -1,
+            1
+          ]
+        ],
+        "title": "VEDA Dashboard Render Parameters"
+      }
+    },
+    "dashboard:is_periodic": true
+  }