Shifted ldl-pusher out of stat-pusher

h3exporter/run_h3exporter.sh

@@ -1,25 +1,31 @@
 #!/usr/bin/env bash
 #### Swarm environment also determined in script, from hostname, for settings
+DEBUG=
+
+# get argument if provided
+if [[ $1 ]]; then
+	if [[ $1 == 'debug' ]]; then
+		DEBUG=1
+	fi
+fi
 
 # setup venv
-export PYTHONPATH=/home/monitor/github/ukwa-monitor/h3exporter
+export PYTHONPATH=~/github/ukwa-monitor/h3exporter
 source ${PYTHONPATH}/venv/bin/activate
 
 # ensure log directory exists
 [[ -d ${PYTHONPATH}/logs/ ]] || mkdir  ${PYTHONPATH}/logs
 
 # ensure python libraries installed
-cd ${PYTHONPATH}
+cd ${PYTHONPATH}/
 pip install -r requirements.txt
 
 # run h3exporter script
-${PYTHONPATH}/h3exporter.py & 
-exit 0
-
+if [[ ${DEBUG} -eq 1 ]]; then
+	${PYTHONPATH}/h3exporter.py &		# if debug argument given, don't push to /dev/null (for live service debugging)
 
-# ------
-if [[ $(hostname -s) =~ ^(prod|monitor) ]]; then
+elif [[ ${HOSTNAME} =~ ^(monitor|prod) ]]; then
 	${PYTHONPATH}/h3exporter.py  > /dev/null &	# disable generation of large logs over time
 else
-	${PYTHONPATH}/h3exporter.py & 
+	${PYTHONPATH}/h3exporter.py &
 fi

ldl-pusher/README.md

@@ -0,0 +1,34 @@
+Requirements
+============
+
+```
+$ python3 -m venv venv
+```
+
+
+LDL VM Monitoring Pusher
+========================
+
+We've always had difficulties tracking the status of DLS LDL VMs, especially their network connectivity.
+
+Each LDL VM has a root cronjob that curls "http://ld02:8983/wa/monitor?host=$(hostname -f)" (NLS and NLW going via 'dls-(bsp|lon)-wb02' instead of directly to 'ld02:8983').  (ld02 being 'lduwka-proxy', our WA infrastructure server that routes all LDL requests onwards to our WA services via Apache httpd proxypasses.)
+
+To set up as a systemctl daemon, as the root user:
+* Copy ldl-pusher service file to systemctl directory
+ * cp ldl-pusher.service /usr/lib/systemd/system/
+
+* Reload systemctl 
+ * systemctl daemon-reload
+
+* Enable and start ldl pusher service
+ * systemctl enable --now ldl-pusher.service
+
+
+Development and Testing
+=======================
+
+The service can also be ran manually. It is best to do this as the monitor/dev user and from the user account home directory as this most simulates the production systemctl service operation.
+
+```
+~$ /home/<user account>/github/ukwa-monitor/ldl-pusher/run_ldl_pusher.sh
+```

ldl-pusher/common/log.py

@@ -0,0 +1,20 @@
+'''
+Common/non-script specific functions
+'''
+
+import logging
+
+def configure(lvl='INFO'):
+	logging.basicConfig(format='[%(asctime)s %(funcName)s %(levelname)s] %(message)s', level=lvl)
+
+def configure_file(eset):
+	scriptHandler = logging.FileHandler(eset['logfpfn'])
+	formatter = logging.Formatter("[%(asctime)s %(funcName)s %(levelname)s]  %(message)s")
+	scriptHandler.setFormatter(formatter)
+	logging.root.addHandler(scriptHandler)
+	logging.root.setLevel(logging.WARNING)
+	if eset['loglevel'] == 'ERROR': logging.getLogger().setLevel(logging.ERROR)
+	elif eset['loglevel'] == 'WARNING': logging.getLogger().setLevel(logging.WARNING)
+	elif eset['loglevel'] == 'DEBUG': logging.getLogger().setLevel(logging.DEBUG)
+	else: logging.getLogger().setLevel(logging.INFO)
+	logger = logging.getLogger('__main__')

stat-pusher/ldl-pusher.py → ldl-pusher/ldl-pusher.py

@@ -6,7 +6,6 @@
 import os, sys, logging
 import socket, re
 import configparser
-import daemon, lockfile
 from http.server import BaseHTTPRequestHandler, HTTPServer
 import datetime
 from prometheus_client import CollectorRegistry, Gauge, push_to_gateway
@@ -16,8 +15,6 @@
 # globals
 logger = logging.getLogger(__name__)
 
-PIDFILE = f"{__file__}.pid"
-LOCKFILE = f"{PIDFILE}.lock"
 SETTINGSFILE = 'settings'
 REQUEST = re.compile("^\w+\s+(/.+)\s+HTTP/\d.\d$")
 HOSTREQ = re.compile("^/wa/monitor\?host=(.+)$")
@@ -186,11 +183,6 @@ def script(eset):
 
 # main ----------------------------------------
 if __name__ == '__main__':
-	# check for lockfile
-	if os.path.exists(LOCKFILE):
-		print(f"Exiting as [{LOCKFILE}] exists, service already be running")
-		sys.exit(1)
-
 	# get swarm environment
 	senvMatch = re.match('^(dev|beta|prod|monitor)', socket.gethostname())
 	if senvMatch:
@@ -203,12 +195,5 @@ def script(eset):
 	# read environment settings
 	eset = _read_settings(environ)
 
-	# run daemon
-	with daemon.DaemonContext(
-		stdout = sys.stdout,
-		stderr = sys.stderr,
-		uid = int(eset['uid']),
-		gid = int(eset['gid']),
-		pidfile = lockfile.FileLock(PIDFILE)
-	):
-		script(eset)
+	# run 
+	script(eset)

stat-pusher/ldl-pusher.service → ldl-pusher/ldl-pusher.service

@@ -6,7 +6,7 @@ After=network.target
 Type=forking
 User=monitor
 Group=monitor
-ExecStart=/home/monitor/github/ukwa-monitor/stat-pusher/run_ldl_pusher.sh
+ExecStart=/home/monitor/github/ukwa-monitor/ldl-pusher/run_ldl_pusher.sh
 Restart=always
 RestartSec=30
 

stat-pusher/ldl-requirements.txt → ldl-pusher/requirements.txt

@@ -1,4 +1,2 @@
-python-daemon
-lockfile
 datetime
 prometheus_client

ldl-pusher/run_ldl_pusher.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+DEBUG=
+
+# get argument if provided
+if [[ $1 ]]; then
+	if [[ $1 == 'debug' ]]; then
+		DEBUG=1
+	fi
+fi
+
+# setup venv
+export PYTHONPATH=~/github/ukwa-monitor/ldl-pusher
+source ${PYTHONPATH}/venv/bin/activate
+
+# ensure log directory exists
+[[ -d ${PYTHONPATH}/logs/ ]] || mkdir ${PYTHONPATH}/logs
+
+# ensure python libraries installed
+cd ${PYTHONPATH}/
+pip install -r requirements.txt
+
+# run ldl-pusher script
+if [[ ${DEBUG} -eq 1 ]]; then
+	${PYTHONPATH}/ldl-pusher.py &		# if debug argument given, don't push to /dev/null (for live service debugging)
+
+elif [[ ${HOSTNAME} =~ ^(monitor|prod) ]]; then
+	${PYTHONPATH}/ldl-pusher.py  > /dev/null &	# disable generation of large logs over time
+else
+	${PYTHONPATH}/ldl-pusher.py &
+fi

ldl-pusher/settings

@@ -0,0 +1,57 @@
+[dev]
+# common
+pushgtw = http://monitor-pushgateway.dapi.wa.bl.uk
+
+# ldl server connection testing
+logfpfn = /home/gilh/github/ukwa-monitor/ldl-pusher/logs/ldl-pusher.log
+output = /home/gilh/github/ukwa-monitor/ldl-pusher/logs/ldl-pusher.out
+loglevel = DEBUG
+uid = 1004
+gid = 1004
+hostname = 0.0.0.0
+port = 9119
+# prometheus settings
+schedule = 10
+job = ldl_rr
+metric = recent_connections
+desc = Curl requests from LDL VMs, indicating LDL to WA connectivity
+
+
+
+[beta]
+# common
+pushgtw = http://monitor-pushgateway.bapi.wa.bl.uk
+
+# ldl server connection testing
+logfpfn = /home/monitor/github/ukwa-monitor/ldl-pusher/logs/ldl-pusher.log
+output = /home/monitor/github/ukwa-monitor/ldl-pusher/logs/ldl-pusher.out
+loglevel = DEBUG
+uid = 1000
+gid = 1000
+hostname = 0.0.0.0
+port = 9119
+# prometheus settings
+schedule = 20
+job = ldl_rr
+metric = recent_connections
+desc = Curl requests from LDL VMs, indicating LDL to WA connectivity
+
+
+
+[prod]
+# common
+pushgtw = http://monitor-pushgateway.api.wa.bl.uk
+
+# ldl server connection testing
+logfpfn = /home/monitor/github/ukwa-monitor/ldl-pusher/logs/ldl-pusher.log
+output = /home/monitor/github/ukwa-monitor/ldl-pusher/logs/ldl-pusher.out
+loglevel = INFO
+uid = 1000
+gid = 1000
+hostname = 0.0.0.0
+port = 9119
+# prometheus settings
+schedule = 20
+job = ldl_rr
+metric = recent_connections
+desc = Curl requests from LDL VMs, indicating LDL to WA connectivity

stat-pusher/README.md

@@ -16,23 +16,3 @@ $ run_stat_pusher.sh dev
 ```
 
 Which should grab the stats and push them to the DEV monitor. It relies on `gitlab/ukwa-monitor` to pick up environment variables.
-
-
-
-LDL VM Monitoring Pusher
-========================
-
-We've always had difficulties tracking the status of DLS LDL VMs, especially their network connectivity.
-
-Each LDL VM has a root cronjob that curls "http://ld02:8983/wa/monitor?host=$(hostname -f)" (NLS and NLW going via 'dls-(bsp|lon)-wb02' instead of directly to 'ld02:8983').  (ld02 being 'lduwka-proxy', our WA infrastructure server that routes all LDL requests onwards to our WA services via Apache httpd proxypasses.)
-
-To set up as a systemctl daemon, as the root user:
-* Copy ldl-pusher service file to systemctl directory
- * cp ldl-pusher.service /usr/lib/systemd/system/
-
-* Reload systemctl 
- * systemctl daemon-reload
-
-* Enable and start ldl pusher service
- * systemctl enable --now ldl-pusher.service
-

stat-pusher/settings

@@ -5,21 +5,6 @@ pushgtw = http://monitor-pushgateway.dapi.wa.bl.uk
 # update_pushgateway_stats.py
 statsfile = dev.stats
 
-# ldl server connection testing
-logfpfn = /home/gilh/github/ukwa-monitor/stat-pusher/logs/ldl-pusher.log
-output = /home/gilh/github/ukwa-monitor/stat-pusher/logs/ldl-pusher.out
-loglevel = DEBUG
-uid = 1004
-gid = 1004
-hostname = 0.0.0.0
-port = 9119
-# prometheus settings
-schedule = 10
-job = ldl_rr
-metric = recent_connections
-desc = Curl requests from LDL VMs, indicating LDL to WA connectivity
-
-
 
 [beta]
 # common
@@ -28,20 +13,6 @@ pushgtw = http://monitor-pushgateway.bapi.wa.bl.uk
 # update_pushgateway_stats.py
 statsfile = beta.stats
 
-# ldl server connection testing
-logfpfn = /home/monitor/github/ukwa-monitor/stat-pusher/logs/ldl-pusher.log
-output = /home/monitor/github/ukwa-monitor/stat-pusher/logs/ldl-pusher.out
-loglevel = DEBUG
-uid = 1000
-gid = 1000
-hostname = 0.0.0.0
-port = 9119
-# prometheus settings
-schedule = 20
-job = ldl_rr
-metric = recent_connections
-desc = Curl requests from LDL VMs, indicating LDL to WA connectivity
-
 
 
 [prod]
@@ -50,17 +21,3 @@ pushgtw = http://monitor-pushgateway.api.wa.bl.uk
 
 # update_pushgateway_stats.py
 statsfile = prod.stats
-
-# ldl server connection testing
-logfpfn = /home/monitor/github/ukwa-monitor/stat-pusher/logs/ldl-pusher.log
-output = /home/monitor/github/ukwa-monitor/stat-pusher/logs/ldl-pusher.out
-loglevel = INFO
-uid = 1000
-gid = 1000
-hostname = 0.0.0.0
-port = 9119
-# prometheus settings
-schedule = 20
-job = ldl_rr
-metric = recent_connections
-desc = Curl requests from LDL VMs, indicating LDL to WA connectivity