diff --git a/.github/workflows/watch-repos.yml b/.github/workflows/watch-repos.yml index 9c7ed68..00da787 100644 --- a/.github/workflows/watch-repos.yml +++ b/.github/workflows/watch-repos.yml @@ -139,100 +139,153 @@ jobs: chmod +x api_helper.sh - name: Full Repository Ingestion if: >- - steps.config.outputs.config_exists == 'true' && github.event_name == - 'workflow_dispatch' && github.event.inputs.full_ingest == 'true' - run: | - source ./api_helper.sh - - echo "Starting full repository ingestion..." - - # Validate required variables - if [ -z "$CONFIG" ]; then - echo "::error::CONFIG variable is empty" - exit 1 - fi - - if [ -z "${{ steps.config.outputs.osiris_url }}" ]; then - echo "::error::osiris_url is not set" - exit 1 - fi - - # Convert CONFIG from YAML to JSON and prepare request - CONFIG_JSON=$(echo "$CONFIG" | yq -o=json '.' | jq -c '.') - TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - - # Initialize ingestion - response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/ingest-repo" "{ - \"repo\": \"${{ github.repository }}\", - \"branch\": \"${{ github.ref_name }}\", - \"metadata\": { - \"repository\": \"${{ github.repository }}\", - \"branch\": \"${{ github.ref_name }}\", - \"event_type\": \"${{ github.event_name }}\", - \"commit_sha\": \"${{ github.sha }}\", - \"process_timestamp\": \"$TIMESTAMP\", - \"config\": $CONFIG_JSON - } - }") - - # Parse response directly without intermediate file - if ! total_batches=$(echo "$response" | jq -r '.totalBatches'); then - echo "::error::Failed to parse response" - echo "$response" - exit 1 - fi - - if [ -z "$total_batches" ] || [ "$total_batches" = "null" ]; then - echo "::error::No batches found in response" - echo "$response" - exit 1 - fi - - echo "Processing $total_batches batches..." + steps.config.outputs.config_exists == 'true' && github.event_name == + 'workflow_dispatch' && github.event.inputs.full_ingest == 'true' + run: > + source ./api_helper.sh + + + echo "Starting full repository ingestion..." + + + # Validate required variables + + if [ -z "$CONFIG" ]; then + echo "::error::CONFIG variable is empty" + exit 1 + fi + + + if [ -z "${{ steps.config.outputs.osiris_url }}" ]; then + echo "::error::osiris_url is not set" + exit 1 + fi + + + # Properly escape and format the CONFIG_JSON + + CONFIG_JSON=$(echo "$CONFIG" | yq -o=json '.' | jq -c '.' | sed + 's/"/\\"/g') + + TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + + + # Create the request body first and validate it + + REQUEST_BODY=$(cat </dev/null 2>&1; then + echo "::error::Invalid JSON request body" + echo "$REQUEST_BODY" + exit 1 + fi + + + # Make the API call + + response=$(call_api "${{ steps.config.outputs.osiris_url + }}/api/ingest-repo" "$REQUEST_BODY") + + + # Parse total batches directly from response + + if ! echo "$response" | jq . >/dev/null 2>&1; then + echo "::error::Invalid JSON response" + echo "$response" + exit 1 + fi + + + total_batches=$(echo "$response" | jq -r '.totalBatches') + + + if [ -z "$total_batches" ] || [ "$total_batches" = "null" ]; then + echo "::error::No batches found in response" + echo "$response" + exit 1 + fi + + + echo "Processing $total_batches batches..." + + + successful_batches=0 + + failed_batches=0 + + + # Process each batch + + for ((batch=0; batch/dev/null 2>&1; then - echo "::warning::Batch $((batch + 1)) returned error:" - echo "$batch_response" | jq '.error' - failed_batches=$((failed_batches + 1)) - else - echo "Batch $((batch + 1)) processed successfully" - successful_batches=$((successful_batches + 1)) - fi - else - echo "::warning::Failed to process batch $((batch + 1))" + if batch_response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/process-batch" "$BATCH_REQUEST"); then + if echo "$batch_response" | jq -e '.error' >/dev/null 2>&1; then + echo "::warning::Batch $((batch + 1)) returned error:" + echo "$batch_response" | jq '.error' failed_batches=$((failed_batches + 1)) + else + echo "Batch $((batch + 1)) processed successfully" + successful_batches=$((successful_batches + 1)) fi - - # Add delay between batches - [ $((batch + 1)) -lt "$total_batches" ] && sleep 2 - done - - # Final status report - echo "Repository ingestion completed" - echo "Summary:" - echo "- Total batches: $total_batches" - echo "- Successful: $successful_batches" - echo "- Failed: $failed_batches" - - if [ "$successful_batches" -eq "$total_batches" ]; then - echo "::notice::Successfully processed all batches" else - echo "::warning::Completed with $failed_batches failed batches" - [ "$successful_batches" -eq 0 ] && exit 1 + echo "::warning::Failed to process batch $((batch + 1))" + failed_batches=$((failed_batches + 1)) fi + + [ $((batch + 1)) -lt "$total_batches" ] && sleep 2 + done + + + # Final status report + + echo "Repository ingestion completed" + + echo "Summary:" + + echo "- Total batches: $total_batches" + + echo "- Successful: $successful_batches" + + echo "- Failed: $failed_batches" + + + if [ "$successful_batches" -eq "$total_batches" ]; then + echo "::notice::Successfully processed all batches" + else + echo "::warning::Completed with $failed_batches failed batches" + [ "$successful_batches" -eq 0 ] && exit 1 + fi - name: Process Incremental Changes if: >- steps.config.outputs.config_exists == 'true' && !(github.event_name