workflow

.github/workflows/watch-repos.yml

@@ -146,87 +146,53 @@ jobs:
             
             echo "Starting full repository ingestion..."
             
-            # Validate required variables and prepare config
-            if [ -z "$CONFIG" ]; then
-              echo "::error::CONFIG variable is empty"
-              exit 1
-            fi
-            
-            if [ -z "${{ steps.config.outputs.osiris_url }}" ]; then
-              echo "::error::osiris_url is not set"
-              exit 1
-            fi
+            TEMP_DIR=$(mktemp -d)
+            trap 'rm -rf "$TEMP_DIR"' EXIT
             
-            # First convert CONFIG to valid JSON and store in a variable
-            CONFIG_JSON=$(echo "$CONFIG" | yq -o=json '.')
+            # Prepare simple request without complex config handling
+            REQUEST_BODY="{
+              \"repo\": \"${{ github.repository }}\",
+              \"branch\": \"${{ github.ref_name }}\",
+              \"metadata\": {
+                \"repository\": \"${{ github.repository }}\",
+                \"branch\": \"${{ github.ref_name }}\",
+                \"event_type\": \"${{ github.event_name }}\",
+                \"commit_sha\": \"${{ github.sha }}\",
+                \"process_timestamp\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\"
+              }
+            }"
             
-            # Create a temporary file for the request body
-            REQUEST_BODY=$(jq -n \
-              --arg repo "${{ github.repository }}" \
-              --arg branch "${{ github.ref_name }}" \
-              --arg event "${{ github.event_name }}" \
-              --arg sha "${{ github.sha }}" \
-              --arg ts "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
-              --argjson cfg "$CONFIG_JSON" \
-              '{
-                repo: $repo,
-                branch: $branch,
-                metadata: {
-                  repository: $repo,
-                  branch: $branch,
-                  event_type: $event,
-                  commit_sha: $sha,
-                  process_timestamp: $ts,
-                  config: $cfg
-                }
-              }')
+            echo "Initializing repository ingestion..."
             
-            # Validate the request body
-            if ! echo "$REQUEST_BODY" | jq . >/dev/null 2>&1; then
-              echo "::error::Invalid JSON request body"
-              echo "$REQUEST_BODY"
-              exit 1
-            fi
-        
             # Make the API call
             response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/ingest-repo" "$REQUEST_BODY")
             
-            # Process response and continue with batches...
-            total_batches=$(echo "$response" | jq -r '.totalBatches')
-            if [ -z "$total_batches" ] || [ "$total_batches" = "null" ]; then
-              echo "::error::No batches found in response"
+            # Process response directly
+            if ! echo "$response" | jq -e '.totalBatches' > /dev/null; then
+              echo "::error::Invalid response from API"
               echo "$response"
               exit 1
             fi
             
+            total_batches=$(echo "$response" | jq -r '.totalBatches')
             echo "Processing $total_batches batches..."
+            
             successful_batches=0
             failed_batches=0
             
-            # Process each batch
             for ((batch=0; batch<total_batches; batch++)); do
               echo "Processing batch $((batch + 1)) of $total_batches"
               
-              batch_body=$(jq -n \
-                --arg repo "${{ github.repository }}" \
-                --arg branch "${{ github.ref_name }}" \
-                --arg start "$((batch * 10))" \
-                '{
-                  repo: $repo,
-                  branch: $branch,
-                  batchStart: ($start|tonumber),
-                  batchSize: 10
-                }')
+              batch_request="{
+                \"repo\": \"${{ github.repository }}\",
+                \"branch\": \"${{ github.ref_name }}\",
+                \"batchStart\": $((batch * 10)),
+                \"batchSize\": 10
+              }"
               
-              if batch_response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/process-batch" "$batch_body"); then
-                if echo "$batch_response" | jq -e '.error' >/dev/null 2>&1; then
-                  echo "::warning::Batch $((batch + 1)) returned error:"
-                  echo "$batch_response" | jq '.error'
-                  failed_batches=$((failed_batches + 1))
-                else
-                  echo "Batch $((batch + 1)) processed successfully"
-                  successful_batches=$((successful_batches + 1))
-                fi
+              if batch_response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/process-batch" "$batch_request"); then
+                echo "Batch $((batch + 1)) processed successfully"
+                successful_batches=$((successful_batches + 1))
               else
                 echo "::warning::Failed to process batch $((batch + 1))"
                 failed_batches=$((failed_batches + 1))
@@ -235,9 +201,7 @@ jobs:
               [ $((batch + 1)) -lt "$total_batches" ] && sleep 2
             done
             
-            # Final status report
-            echo "Repository ingestion completed"
-            echo "Summary:"
+            echo "Repository ingestion completed:"
             echo "- Total batches: $total_batches"
             echo "- Successful: $successful_batches"
             echo "- Failed: $failed_batches"