From 5db4348f15711dbc0aea318b1bebe2e7edb855d2 Mon Sep 17 00:00:00 2001
From: ivan <ivanminutillo@outlook.it>
Date: Wed, 8 Jan 2025 15:23:03 +0100
Subject: [PATCH] workflow

---
 .github/workflows/watch-repos.yml | 201 +++++++++---------------------
 1 file changed, 62 insertions(+), 139 deletions(-)

diff --git a/.github/workflows/watch-repos.yml b/.github/workflows/watch-repos.yml
index 86d0f65..56131a6 100644
--- a/.github/workflows/watch-repos.yml
+++ b/.github/workflows/watch-repos.yml
@@ -142,150 +142,73 @@ jobs:
           chmod +x api_helper.sh
       - name: Full Repository Ingestion
         if: >-
-          steps.config.outputs.config_exists == 'true' &&  github.event_name ==
-          'workflow_dispatch' &&  github.event.inputs.full_ingest == 'true'
-        run: >
-          source ./api_helper.sh
-
-
-          echo "Starting full repository ingestion..."
-
-
-          TEMP_DIR=$(mktemp -d)
-
-          trap 'rm -rf "$TEMP_DIR"' EXIT
-
-
-          echo "DEBUG: Current CONFIG content:"
-
-          echo "$CONFIG" | jq '.'
-
-
-          # Build request body using jq for safe JSON construction
-
-          REQUEST_BODY=$(jq -n \
-            --arg repo "${{ github.repository }}" \
-            --arg branch "${{ github.ref_name }}" \
-            --arg event_type "${{ github.event_name }}" \
-            --arg commit_sha "${{ github.sha }}" \
-            --arg timestamp "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
-            --argjson config "$CONFIG" \
-          '{
-            "repo": $repo,
-            "branch": $branch,
-            "metadata": {
-              "repository": $repo,
-              "branch": $branch,
-              "event_type": $event_type,
-              "commit_sha": $commit_sha,
-              "process_timestamp": $timestamp,
-              "config": $config
-            },
-            "maxFileSize": ($config.max_file_size // 100000),
-            "maxTokens": ($config.max_tokens // 50000),
-            "forceReplace": true
-          }')
-
-
-          echo "DEBUG: Request body before sending:"
-
-          echo "$REQUEST_BODY" | jq '.'
-
-
-          # Make the API call and capture response
-
-          echo "Making API call to ingest-repo..."
-
-          response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/ingest-repo" "$REQUEST_BODY")
-          api_status=$?
-
-          if [ $api_status -ne 0 ]; then
-              echo "::error::API call failed"
-              exit 1
-          fi
-
-
-          echo "DEBUG: Raw API Response:"
-
-          echo "$response"
-
-
-          # Validate response is valid JSON
-
-          if ! echo "$response" | jq '.' > /dev/null 2>&1; then
-            echo "::error::Invalid JSON response from API"
-            echo "$response"
-            exit 1
-          fi
-
-          # Parse response for totalBatches
-          total_batches=$(echo "$response" | jq -r '.totalBatches')
-          if [ -z "$total_batches" ] || [ "$total_batches" = "null" ]; then
-              echo "::error::Could not extract totalBatches from response"
-              echo "Response was: $response"
-              exit 1
-          fi
-
-
-          echo "Processing $total_batches batches..."
-
-          successful_batches=0
-
-          failed_batches=0
-
-
-          # Process batches
-
-          for ((batch=0; batch<total_batches; batch++)); do
-            echo "Processing batch $((batch + 1)) of $total_batches"
+            steps.config.outputs.config_exists == 'true' &&  github.event_name == 'workflow_dispatch' &&  github.event.inputs.full_ingest == 'true'
+        run: |
+            source ./api_helper.sh
+        
+            echo "Starting full repository ingestion..."
+            echo "DEBUG: Making initial API call..."
+            
+            # Make the API call - response is already validated by call_api
+            response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/ingest-repo" "$REQUEST_BODY")
+            api_status=$?
             
-            batch_request=$(jq -n \
-              --arg repo "${{ github.repository }}" \
-              --arg branch "${{ github.ref_name }}" \
-              --argjson start "$((batch * 10))" \
-              --argjson size "10" \
-            '{
-              "repo": $repo,
-              "branch": $branch,
-              "batchStart": $start,
-              "batchSize": $size
-            }')
+            if [ $api_status -ne 0 ]; then
+                echo "::error::API call failed"
+                exit 1
+            fi
             
-            echo "DEBUG: Batch request body:"
-            echo "$batch_request" | jq '.'
+            # Extract totalBatches directly (no need to revalidate JSON)
+            total_batches=$(echo "$response" | jq -r '.totalBatches')
             
-            if batch_response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/process-batch" "$batch_request"); then
-              echo "DEBUG: Batch response:"
-              echo "$batch_response" | jq '.'
-              echo "Batch $((batch + 1)) processed successfully"
-              successful_batches=$((successful_batches + 1))
+            if [ "$total_batches" = "null" ] || [ -z "$total_batches" ]; then
+                echo "::error::Could not find totalBatches in response"
+                echo "Response was: $response"
+                exit 1
+            fi
+        
+            echo "Processing $total_batches batches..."
+            successful_batches=0
+            failed_batches=0
+        
+            # Process batches
+            for ((batch=0; batch<total_batches; batch++)); do
+                echo "Processing batch $((batch + 1)) of $total_batches"
+                batch_request=$(jq -n \
+                    --arg repo "${{ github.repository }}" \
+                    --arg branch "${{ github.ref_name }}" \
+                    --argjson batchStart "$((batch * 10))" \
+                    --argjson batchSize "10" \
+                    '{
+                        repo: $repo,
+                        branch: $branch,
+                        batchStart: $batchStart,
+                        batchSize: $batchSize
+                    }'
+                )
+                
+                if batch_response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/process-batch" "$batch_request"); then
+                    echo "Batch $((batch + 1)) processed successfully"
+                    successful_batches=$((successful_batches + 1))
+                else
+                    echo "::warning::Failed to process batch $((batch + 1))"
+                    failed_batches=$((failed_batches + 1))
+                fi
+                
+                [ $((batch + 1)) -lt "$total_batches" ] && sleep 2
+            done
+        
+            echo "Repository ingestion completed:"
+            echo "- Total batches: $total_batches"
+            echo "- Successful: $successful_batches"
+            echo "- Failed: $failed_batches"
+        
+            if [ "$successful_batches" -eq "$total_batches" ]; then
+                echo "::notice::Successfully processed all batches"
             else
-              echo "::warning::Failed to process batch $((batch + 1))"
-              echo "DEBUG: Failed batch response:"
-              echo "$batch_response"
-              failed_batches=$((failed_batches + 1))
+                echo "::warning::Completed with $failed_batches failed batches"
+                [ "$successful_batches" -eq 0 ] && exit 1
             fi
-            
-            # Add small delay between batches
-            [ $((batch + 1)) -lt "$total_batches" ] && sleep 2
-          done
-
-
-          echo "Repository ingestion completed:"
-
-          echo "- Total batches: $total_batches"
-
-          echo "- Successful: $successful_batches"
-
-          echo "- Failed: $failed_batches"
-
-
-          if [ "$successful_batches" -eq "$total_batches" ]; then
-            echo "::notice::Successfully processed all batches"
-          else
-            echo "::warning::Completed with $failed_batches failed batches"
-            [ "$successful_batches" -eq 0 ] && exit 1
-          fi
       - name: Process Incremental Changes
         if: >-
           steps.config.outputs.config_exists == 'true' &&  !(github.event_name