Skip to content

Commit

Permalink
workflow
Browse files Browse the repository at this point in the history
  • Loading branch information
@ivanminutillo
ivanminutillo committed Jan 8, 2025
1 parent f96608c commit 5db4348
Showing 1 changed file with 62 additions and 139 deletions.
201 changes: 62 additions & 139 deletions .github/workflows/watch-repos.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,150 +142,73 @@ jobs:
chmod +x api_helper.sh
- name: Full Repository Ingestion
if: >-
steps.config.outputs.config_exists == 'true' && github.event_name ==
'workflow_dispatch' && github.event.inputs.full_ingest == 'true'
run: >
source ./api_helper.sh
echo "Starting full repository ingestion..."
TEMP_DIR=$(mktemp -d)
trap 'rm -rf "$TEMP_DIR"' EXIT
echo "DEBUG: Current CONFIG content:"
echo "$CONFIG" | jq '.'
# Build request body using jq for safe JSON construction
REQUEST_BODY=$(jq -n \
--arg repo "${{ github.repository }}" \
--arg branch "${{ github.ref_name }}" \
--arg event_type "${{ github.event_name }}" \
--arg commit_sha "${{ github.sha }}" \
--arg timestamp "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
--argjson config "$CONFIG" \
'{
"repo": $repo,
"branch": $branch,
"metadata": {
"repository": $repo,
"branch": $branch,
"event_type": $event_type,
"commit_sha": $commit_sha,
"process_timestamp": $timestamp,
"config": $config
},
"maxFileSize": ($config.max_file_size // 100000),
"maxTokens": ($config.max_tokens // 50000),
"forceReplace": true
}')
echo "DEBUG: Request body before sending:"
echo "$REQUEST_BODY" | jq '.'
# Make the API call and capture response
echo "Making API call to ingest-repo..."
response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/ingest-repo" "$REQUEST_BODY")
api_status=$?
if [ $api_status -ne 0 ]; then
echo "::error::API call failed"
exit 1
fi
echo "DEBUG: Raw API Response:"
echo "$response"
# Validate response is valid JSON
if ! echo "$response" | jq '.' > /dev/null 2>&1; then
echo "::error::Invalid JSON response from API"
echo "$response"
exit 1
fi
# Parse response for totalBatches
total_batches=$(echo "$response" | jq -r '.totalBatches')
if [ -z "$total_batches" ] || [ "$total_batches" = "null" ]; then
echo "::error::Could not extract totalBatches from response"
echo "Response was: $response"
exit 1
fi
echo "Processing $total_batches batches..."
successful_batches=0
failed_batches=0
# Process batches
for ((batch=0; batch<total_batches; batch++)); do
echo "Processing batch $((batch + 1)) of $total_batches"
steps.config.outputs.config_exists == 'true' && github.event_name == 'workflow_dispatch' && github.event.inputs.full_ingest == 'true'
run: |
source ./api_helper.sh
echo "Starting full repository ingestion..."
echo "DEBUG: Making initial API call..."
# Make the API call - response is already validated by call_api
response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/ingest-repo" "$REQUEST_BODY")
api_status=$?
batch_request=$(jq -n \
--arg repo "${{ github.repository }}" \
--arg branch "${{ github.ref_name }}" \
--argjson start "$((batch * 10))" \
--argjson size "10" \
'{
"repo": $repo,
"branch": $branch,
"batchStart": $start,
"batchSize": $size
}')
if [ $api_status -ne 0 ]; then
echo "::error::API call failed"
exit 1
fi
echo "DEBUG: Batch request body:"
echo "$batch_request" | jq '.'
# Extract totalBatches directly (no need to revalidate JSON)
total_batches=$(echo "$response" | jq -r '.totalBatches')
if batch_response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/process-batch" "$batch_request"); then
echo "DEBUG: Batch response:"
echo "$batch_response" | jq '.'
echo "Batch $((batch + 1)) processed successfully"
successful_batches=$((successful_batches + 1))
if [ "$total_batches" = "null" ] || [ -z "$total_batches" ]; then
echo "::error::Could not find totalBatches in response"
echo "Response was: $response"
exit 1
fi
echo "Processing $total_batches batches..."
successful_batches=0
failed_batches=0
# Process batches
for ((batch=0; batch<total_batches; batch++)); do
echo "Processing batch $((batch + 1)) of $total_batches"
batch_request=$(jq -n \
--arg repo "${{ github.repository }}" \
--arg branch "${{ github.ref_name }}" \
--argjson batchStart "$((batch * 10))" \
--argjson batchSize "10" \
'{
repo: $repo,
branch: $branch,
batchStart: $batchStart,
batchSize: $batchSize
}'
)
if batch_response=$(call_api "${{ steps.config.outputs.osiris_url }}/api/process-batch" "$batch_request"); then
echo "Batch $((batch + 1)) processed successfully"
successful_batches=$((successful_batches + 1))
else
echo "::warning::Failed to process batch $((batch + 1))"
failed_batches=$((failed_batches + 1))
fi
[ $((batch + 1)) -lt "$total_batches" ] && sleep 2
done
echo "Repository ingestion completed:"
echo "- Total batches: $total_batches"
echo "- Successful: $successful_batches"
echo "- Failed: $failed_batches"
if [ "$successful_batches" -eq "$total_batches" ]; then
echo "::notice::Successfully processed all batches"
else
echo "::warning::Failed to process batch $((batch + 1))"
echo "DEBUG: Failed batch response:"
echo "$batch_response"
failed_batches=$((failed_batches + 1))
echo "::warning::Completed with $failed_batches failed batches"
[ "$successful_batches" -eq 0 ] && exit 1
fi
# Add small delay between batches
[ $((batch + 1)) -lt "$total_batches" ] && sleep 2
done
echo "Repository ingestion completed:"
echo "- Total batches: $total_batches"
echo "- Successful: $successful_batches"
echo "- Failed: $failed_batches"
if [ "$successful_batches" -eq "$total_batches" ]; then
echo "::notice::Successfully processed all batches"
else
echo "::warning::Completed with $failed_batches failed batches"
[ "$successful_batches" -eq 0 ] && exit 1
fi
- name: Process Incremental Changes
if: >-
steps.config.outputs.config_exists == 'true' && !(github.event_name
Expand Down

0 comments on commit 5db4348

Please sign in to comment.