Merge pull request #42 from GilHoggarth/master

LDL connection monitoring
ukwa · Nov 17, 2022 · e45d048 · e45d048
2 parents 38839ba + b5153ff
commit e45d048
Show file tree

Hide file tree

Showing 10 changed files with 346 additions and 3,627 deletions.
diff --git a/.gitignore b/.gitignore
@@ -7,6 +7,12 @@ ENV/
 
 *.pyc
 *out
+*log
+*pid
+*lock
+stat-pusher/dev[0-9]-*-[0-9]*
+stat-pusher/beta[0-9]-*-[0-9]*
+stat-pusher/prod[0-9]-*-[0-9]*
 
 */alertmanager/config.yml
 */grafana/grafana.ini

diff --git a/monitor/dev/start_monitor.sh b/monitor/dev/start_monitor.sh
@@ -9,7 +9,7 @@ export VISUALIZER_PORT=8081
 export GRAFANA_PORT=3000
 export PROMETHEUS_PORT=9090
 export ALERTMANAGER_PORT=9093
-export DATA_PREFIX=/mnt/nfs/data/monitor
+export DATA_PREFIX=/mnt/nfs/data/gilh
 export DATA_GRAFANA=${DATA_PREFIX}/ukwa-monitor/grafana
 export DATA_PROMETHEUS=${DATA_PREFIX}/ukwa-monitor/prometheus
 export DATA_ALERTMANAGER=${DATA_PREFIX}/ukwa-monitor/alertmanager

diff --git a/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py b/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py
@@ -147,6 +147,9 @@ def replace_output_single(outHandle, **kwargs):
 		elif kwargs['title'] == 'Used':
 			expr = 'round(100 - hadoop_hdfs_namenode_nninfo_percent_remaining{instance=\\"${HDFS3_EXPORTER}\\"})'
 			templateCode = templateCode.replace('<expr>', expr)
+	elif kwargs['title'] == 'LDLs':
+		expr = 'recent_connections{instance=\\"ldl_connection_count\\", job=\\"ldl_rr\\"}'
+		templateCode = templateCode.replace('<expr>', expr)
 
 	# add last comma if not last panel
 	if 'lastPanel' not in kwargs:
@@ -207,6 +210,7 @@ def main():
 	replace_output_single(outHandle, pnl=panelStat, job='infrastructure', title='CPU', h=2, w=2, x=2, y=9)
 	replace_output_single(outHandle, pnl=panelStat, job='infrastructure', title='Dsk', h=2, w=2, x=4, y=9)
 	replace_output_single(outHandle, pnl=panelStat, job = 'infrastructure', title = 'Mem', h=2, w=2, x=6, y=9)
+	replace_output_single(outHandle, pnl=panelStat, job='infrastructure', title='LDLs', h=2, w=2, x=0, y=11, textmode='value', colour1='#D44A3A', colour3='#299C46', threshold2='8.5', threshold3='9.5')
 	# general services
 	replace_output_title(outHandle, pnl=panelTitle, job='services', title='Services', h=1, w=8, x=8, y=8)
 	replace_output_single(outHandle, pnl=panelStat, job='services', title='Up', h=2, w=2, x=8, y=9)

diff --git a/monitor/grafana/provisioning/dashboards/daily_dashboard.json b/monitor/grafana/provisioning/dashboards/daily_dashboard.json
@@ -436,7 +436,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "hdfs_under_replicated_block_count{instance='hdfs-exporter.bapi.wa.bl.uk:80'}",
+          "expr": "hdfs_under_replicated_block_count{instance='hdfs-exporter.dapi.wa.bl.uk:80'}",
           "format": "time_series",
           "instant": true,
           "intervalFactor": 1,
@@ -521,7 +521,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "hdfs_used_percent{instance='hdfs-exporter.bapi.wa.bl.uk:80'}",
+          "expr": "hdfs_used_percent{instance='hdfs-exporter.dapi.wa.bl.uk:80'}",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "",
@@ -602,7 +602,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "hdfs_node_count{status='dead',instance='hdfs-exporter.bapi.wa.bl.uk:80'}",
+          "expr": "hdfs_node_count{status='dead',instance='hdfs-exporter.dapi.wa.bl.uk:80'}",
           "format": "time_series",
           "instant": true,
           "intervalFactor": 1,
@@ -683,7 +683,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "hdfs_node_count{status='live',instance='hdfs-exporter.bapi.wa.bl.uk:80'}",
+          "expr": "hdfs_node_count{status='live',instance='hdfs-exporter.dapi.wa.bl.uk:80'}",
           "format": "time_series",
           "instant": true,
           "intervalFactor": 1,