From fcab08ab0a263ca4f633462abbb40f286ce578ab Mon Sep 17 00:00:00 2001
From: Rub21 <rub2106@gmail.com>
Date: Sat, 4 Jan 2025 07:44:01 -0500
Subject: [PATCH] Update liveness for minute replication job

---
 images/replication-job/liveness.sh | 43 +++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 4 deletions(-)

diff --git a/images/replication-job/liveness.sh b/images/replication-job/liveness.sh
index c343c24e..7b230987 100755
--- a/images/replication-job/liveness.sh
+++ b/images/replication-job/liveness.sh
@@ -1,9 +1,44 @@
 #!/usr/bin/env bash
-# This is a script for the complex evaluation of whether Osmosis or other processes are running in the container.
-if [ $(ps -ef | grep -E 'java' | grep -v grep | wc -l) -ge 1 ]; then
+# Script to check if Osmosis (Java) is running and also check how old processed_files.log is.
+# If processed_files.log is older than MAX_AGE_MINUTES, then kill Osmosis.
+
+LOG_FILE="/mnt/data/processed_files.log"
+MAX_AGE_MINUTES=10
+
+get_file_age_in_minutes() {
+  local file="$1"
+  if [ ! -f "$file" ]; then
+    echo 999999
+    return
+  fi
+  local now
+  local mtime
+  now=$(date +%s)
+  mtime=$(stat -c %Y "$file")
+  local diff=$(( (now - mtime) / 60 )) 
+  echo "$diff"
+}
+
+# Check if Osmosis (Java) is running
+OSMOSIS_COUNT=$(ps -ef | grep -E 'java.*osmosis' | grep -v grep | wc -l)
+
+if [ "$OSMOSIS_COUNT" -ge 1 ]; then
   echo "Osmosis is running."
-  exit 0
+  # Check how old the processed_files.log file is
+  file_age=$(get_file_age_in_minutes "$LOG_FILE")
+  echo "processed_files.log file age in minutes: $file_age"
+  if [ "$file_age" -ge "$MAX_AGE_MINUTES" ]; then
+    echo "processed_files.log is older than $MAX_AGE_MINUTES minutes. Attempting to kill Osmosis and restart the container..."
+    # Kill the Osmosis process
+    pkill -f "java.*osmosis" || true
+    echo "Osmosis is not terminating. Force-killing the container..."
+    echo "Container force-restart triggered."
+    exit 2
+  else
+    echo "processed_files.log is not too old. No action needed."
+    exit 0
+  fi
 else
-  echo "Osmosis is not running!" 1>&2
+  echo "Osmosis is not running!"
   exit 1
 fi