diff --git a/scripts/connector-keepalive.sh b/scripts/connector-keepalive.sh index 6af3634..2657eac 100755 --- a/scripts/connector-keepalive.sh +++ b/scripts/connector-keepalive.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash if ! command -v jq &> /dev/null; then echo "Error: jq is not installed." @@ -16,7 +16,6 @@ ENDPOINTS=( "https://axiom-dev.ddn.hasura.app/graphql" ) -# Define the query QUERY='{ "query": "query buildUserDashboard { formatDateToIso(dateString: \"2025-01-01\") usersById(id: 1) { email } customers(limit: 1, where: {segment: {_eq: \"family\"}}) { firstName lastName email segment customerLinks { customerPreferences { socialMedia { linkedin } } supportDB { supportHistory { date status } } } creditCards(where: {expiry: {_gte: \"2024-01-01\"}}) { maskCreditCard expiry cvv } billings(where: {billingDate: {_lte: \"2025-01-01\"}}) { formatBillingDate paymentStatus totalAmount } } calls(limit: 1) { callid } cdr(limit: 1) { guid } documents(limit: 1) { uuid } }", "operationName": "buildUserDashboard" }' @@ -25,47 +24,65 @@ TESTQUERY='{ "query": "query simpleUserRetrieval { customers(limit: 1) { customerId } }", "operationName": "simpleUserRetrieval" }' +ERROR_FILE="/tmp/axiom_error_counts.json" + +if [[ ! -f "$ERROR_FILE" ]]; then + echo '{}' > "$ERROR_FILE" +fi + +ERROR_COUNTS=$(cat "$ERROR_FILE") +ERROR_LIMIT=5 + # Function to get current time in milliseconds current_time_ms() { if [[ "$OSTYPE" == "darwin"* ]]; then - # macOS: use gdate from coreutils gdate +%s%3N else - # Linux: use date date +%s%3N fi } notify_slack() { local message=$1 - payload=$(jq -n --arg text "$message" '{text: $text}') curl -s -X POST -H "Content-Type: application/json" -d "$payload" "$SLACK_WEBHOOK" } for ENDPOINT in "${ENDPOINTS[@]}"; do - { - # Define locally to prevent parallel execution from changing it for other processes - query="$QUERY" - if [[ "$ENDPOINT" =~ https://axiom-(test|dev).ddn.hasura.app/graphql ]]; then - query="$TESTQUERY" - fi + query="$QUERY" + if [[ "$ENDPOINT" =~ https://axiom-(test|dev).ddn.hasura.app/graphql ]]; then + query="$TESTQUERY" + fi + START_TIME=$(current_time_ms) + RESPONSE=$(curl -s -w "\nHTTP_CODE:%{http_code}" -X POST -H "Content-Type: application/json" -d "$query" "$ENDPOINT") + END_TIME=$(current_time_ms) + + HTTP_CODE=$(echo "$RESPONSE" | tail -n 1 | sed 's/HTTP_CODE://') + RESPONSE_BODY=$(echo "$RESPONSE" | sed '$d') + DURATION=$((END_TIME - START_TIME)) + + MESSAGE="Endpoint: $ENDPOINT | HTTP Code: $HTTP_CODE | Time Taken: ${DURATION}ms | Response: $RESPONSE_BODY" + echo "$MESSAGE" - START_TIME=$(current_time_ms) - RESPONSE=$(curl -s -w "\nHTTP_CODE:%{http_code}" -X POST -H "Content-Type: application/json" -d "$query" "$ENDPOINT") - END_TIME=$(current_time_ms) + ERROR_COUNT=$(echo "$ERROR_COUNTS" | jq -r --arg ENDPOINT "$ENDPOINT" '.[$ENDPOINT] // 0') - HTTP_CODE=$(echo "$RESPONSE" | tail -n 1 | sed 's/HTTP_CODE://') - RESPONSE_BODY=$(echo "$RESPONSE" | sed '$d') - DURATION=$((END_TIME - START_TIME)) + if echo "$RESPONSE_BODY" | grep -qi "error"; then + ERROR_COUNT=$((ERROR_COUNT + 1)) - MESSAGE="Endpoint: $ENDPOINT | HTTP Code: $HTTP_CODE | Time Taken: ${DURATION}ms | Response: $RESPONSE_BODY" - echo "$MESSAGE" + # Update error file + ERROR_COUNTS=$(echo "$ERROR_COUNTS" | jq --arg ENDPOINT "$ENDPOINT" --argjson COUNT "$ERROR_COUNT" '. + {($ENDPOINT): $COUNT}') + echo "$ERROR_COUNTS" > "$ERROR_FILE" - if echo "$RESPONSE_BODY" | grep -qi "error"; then - notify_slack "$MESSAGE" + if [[ "$ERROR_COUNT" -ge "$ERROR_LIMIT" ]]; then + notify_slack "⚠️ Consecutive errors detected ($ERROR_COUNT) for $ENDPOINT! Last response: $MESSAGE" + fi + else + if [[ "$ERROR_COUNT" -ge "$ERROR_LIMIT" ]]; then + notify_slack "✅ $ENDPOINT has recovered after $ERROR_COUNT failures." fi - } & -done -wait \ No newline at end of file + # Reset error count + ERROR_COUNTS=$(echo "$ERROR_COUNTS" | jq --arg ENDPOINT "$ENDPOINT" '. + {($ENDPOINT): 0}') + echo "$ERROR_COUNTS" > "$ERROR_FILE" + fi +done \ No newline at end of file