Improves the keepalive to only ping slack after 5 consecutive errors

scripts/connector-keepalive.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 if ! command -v jq &> /dev/null; then
   echo "Error: jq is not installed."
@@ -16,7 +16,6 @@ ENDPOINTS=(
   "https://axiom-dev.ddn.hasura.app/graphql"
 )
 
-# Define the query
 QUERY='{
   "query": "query buildUserDashboard { formatDateToIso(dateString: \"2025-01-01\") usersById(id: 1) { email } customers(limit: 1, where: {segment: {_eq: \"family\"}}) { firstName lastName email segment customerLinks { customerPreferences { socialMedia { linkedin } } supportDB { supportHistory { date status } } } creditCards(where: {expiry: {_gte: \"2024-01-01\"}}) { maskCreditCard expiry cvv } billings(where: {billingDate: {_lte: \"2025-01-01\"}}) { formatBillingDate paymentStatus totalAmount } } calls(limit: 1) { callid } cdr(limit: 1) { guid } documents(limit: 1) { uuid } }", "operationName": "buildUserDashboard"
 }'
@@ -25,47 +24,65 @@ TESTQUERY='{
   "query": "query simpleUserRetrieval { customers(limit: 1) { customerId } }", "operationName": "simpleUserRetrieval"
 }'
 
+ERROR_FILE="/tmp/axiom_error_counts.json"
+
+if [[ ! -f "$ERROR_FILE" ]]; then
+  echo '{}' > "$ERROR_FILE"
+fi
+
+ERROR_COUNTS=$(cat "$ERROR_FILE")
+ERROR_LIMIT=5
+
 # Function to get current time in milliseconds
 current_time_ms() {
   if [[ "$OSTYPE" == "darwin"* ]]; then
-    # macOS: use gdate from coreutils
     gdate +%s%3N
   else
-    # Linux: use date
     date +%s%3N
   fi
 }
 
 notify_slack() {
   local message=$1
-
   payload=$(jq -n --arg text "$message" '{text: $text}')
   curl -s -X POST -H "Content-Type: application/json" -d "$payload" "$SLACK_WEBHOOK"
 }
 
 for ENDPOINT in "${ENDPOINTS[@]}"; do
-  {
-    # Define locally to prevent parallel execution from changing it for other processes
-    query="$QUERY"
-    if [[ "$ENDPOINT" =~ https://axiom-(test|dev).ddn.hasura.app/graphql ]]; then
-      query="$TESTQUERY"
-    fi
+  query="$QUERY"
+  if [[ "$ENDPOINT" =~ https://axiom-(test|dev).ddn.hasura.app/graphql ]]; then
+    query="$TESTQUERY"
+  fi
+  START_TIME=$(current_time_ms)
+  RESPONSE=$(curl -s -w "\nHTTP_CODE:%{http_code}" -X POST -H "Content-Type: application/json" -d "$query" "$ENDPOINT")
+  END_TIME=$(current_time_ms)
+
+  HTTP_CODE=$(echo "$RESPONSE" | tail -n 1 | sed 's/HTTP_CODE://')
+  RESPONSE_BODY=$(echo "$RESPONSE" | sed '$d')
+  DURATION=$((END_TIME - START_TIME))
+
+  MESSAGE="Endpoint: $ENDPOINT | HTTP Code: $HTTP_CODE | Time Taken: ${DURATION}ms | Response: $RESPONSE_BODY"
+  echo "$MESSAGE"
 
-    START_TIME=$(current_time_ms)
-    RESPONSE=$(curl -s -w "\nHTTP_CODE:%{http_code}" -X POST -H "Content-Type: application/json" -d "$query" "$ENDPOINT")
-    END_TIME=$(current_time_ms)
+  ERROR_COUNT=$(echo "$ERROR_COUNTS" | jq -r --arg ENDPOINT "$ENDPOINT" '.[$ENDPOINT] // 0')
 
-    HTTP_CODE=$(echo "$RESPONSE" | tail -n 1 | sed 's/HTTP_CODE://')
-    RESPONSE_BODY=$(echo "$RESPONSE" | sed '$d')
-    DURATION=$((END_TIME - START_TIME))
+  if echo "$RESPONSE_BODY" | grep -qi "error"; then
+    ERROR_COUNT=$((ERROR_COUNT + 1))
 
-    MESSAGE="Endpoint: $ENDPOINT | HTTP Code: $HTTP_CODE | Time Taken: ${DURATION}ms | Response: $RESPONSE_BODY"
-    echo "$MESSAGE"
+    # Update error file
+    ERROR_COUNTS=$(echo "$ERROR_COUNTS" | jq --arg ENDPOINT "$ENDPOINT" --argjson COUNT "$ERROR_COUNT" '. + {($ENDPOINT): $COUNT}')
+    echo "$ERROR_COUNTS" > "$ERROR_FILE"
 
-    if echo "$RESPONSE_BODY" | grep -qi "error"; then
-      notify_slack "$MESSAGE"
+    if [[ "$ERROR_COUNT" -ge "$ERROR_LIMIT" ]]; then
+      notify_slack "⚠️ Consecutive errors detected ($ERROR_COUNT) for $ENDPOINT! Last response: $MESSAGE"
+    fi
+  else
+    if [[ "$ERROR_COUNT" -ge "$ERROR_LIMIT" ]]; then
+      notify_slack "✅ $ENDPOINT has recovered after $ERROR_COUNT failures."
     fi
-  } &
-done
 
-wait
+    # Reset error count
+    ERROR_COUNTS=$(echo "$ERROR_COUNTS" | jq --arg ENDPOINT "$ENDPOINT" '. + {($ENDPOINT): 0}')
+    echo "$ERROR_COUNTS" > "$ERROR_FILE"
+  fi
+done