Merge pull request #13 from GilHoggarth/master

Trackdb alert and reporting - it's all great...
ukwa · Aug 18, 2020 · 10e6ee5 · 10e6ee5
2 parents 8e23af4 + 85a0c62
commit 10e6ee5
Show file tree

Hide file tree

Showing 14 changed files with 479 additions and 32 deletions.
diff --git a/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py b/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py
@@ -105,6 +105,9 @@ def replace_output_single(outHandle, **kwargs):
 	elif kwargs['title'] == 'WWW' or kwargs['title'] == 'Query':
 		expr = 'count(probe_http_status_code{job=\\"' + kwargs['job'] + '\\"} != 200) OR vector(0)'
 		templateCode = templateCode.replace('<expr>', expr)
+	elif kwargs['title'] == 'trackdb':
+		expr = '((time() - trackdb_refresh_timestamp) / (60*60) > 24) OR vector(0)'
+		templateCode = templateCode.replace('<expr>', expr)
 
 	# remove last comma if last panel
 	if 'lastPanel' in kwargs:
@@ -155,6 +158,7 @@ def main():
 	replace_output_single(outHandle, tmpFl=panelSingle, job='services', title='CPU', h=2, w=2, x=2, y=6)
 	replace_output_single(outHandle, tmpFl=panelSingle, job='services', title='Dsk', h=2, w=2, x=4, y=6)
 	replace_output_single(outHandle, tmpFl=panelSingle, job='services', title='Mem', h=2, w=2, x=6, y=6)
+	replace_output_single(outHandle, tmpFl=panelSingle, job='services', title='trackdb', h=2, w=2, x=0, y=8)
 	replace_output_title(outHandle, tmpFl=panelTitle, job='gluster', title='Gluster', h=1, w=8, x=8, y=6)
 	replace_output_single(outHandle, tmpFl=panelSingle, job='gluster', title='Up', h=2, w=2, x=8, y=7)
 	replace_output_single(outHandle, tmpFl=panelSingle, job='gluster', title='CPU', h=2, w=2, x=10, y=7)

diff --git a/monitor/grafana/provisioning/dashboards/wa_status.json b/monitor/grafana/provisioning/dashboards/wa_status.json
@@ -1812,6 +1812,85 @@
       ],
       "valueName": "current"
     },
+    {
+      "cacheTimeout": null,
+      "colorBackground": true,
+      "colorValue": false,
+      "colors": [
+        "#299c46",
+        "rgba(237, 129, 40, 0.89)",
+        "#d44a3a"
+      ],
+      "format": "none",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 2,
+        "w": 2,
+        "x": 0,
+        "y": 8
+      },
+      "id": 27,
+      "interval": null,
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": false
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "expr": "((time() - trackdb_refresh_timestamp) / (60*60) > 24) OR vector(0)",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "refId": "A"
+        }
+      ],
+      "thresholds": "0.1,1",
+      "title": "trackdb",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "current"
+    },
     { 
       "content": "",
       "gridPos": {
@@ -1820,7 +1899,7 @@
         "x": 8,
         "y": 6
       },
-      "id": 27,
+      "id": 28,
       "links": [],
       "mode": "markdown",
       "title": "Gluster",
@@ -1849,7 +1928,7 @@
         "x": 8,
         "y": 7
       },
-      "id": 28,
+      "id": 29,
       "interval": null,
       "links": [],
       "mappingType": 1,
@@ -1928,7 +2007,7 @@
         "x": 10,
         "y": 7
       },
-      "id": 29,
+      "id": 30,
       "interval": null,
       "links": [],
       "mappingType": 1,
@@ -2007,7 +2086,7 @@
         "x": 12,
         "y": 7
       },
-      "id": 30,
+      "id": 31,
       "interval": null,
       "links": [],
       "mappingType": 1,
@@ -2086,7 +2165,7 @@
         "x": 14,
         "y": 7
       },
-      "id": 31,
+      "id": 32,
       "interval": null,
       "links": [],
       "mappingType": 1,
@@ -2150,7 +2229,7 @@
         "x": 16,
         "y": 5
       },
-      "id": 32,
+      "id": 33,
       "links": [],
       "mode": "markdown",
       "title": "Solr",
@@ -2179,7 +2258,7 @@
         "x": 16,
         "y": 6
       },
-      "id": 33,
+      "id": 34,
       "interval": null,
       "links": [],
       "mappingType": 1,
@@ -2258,7 +2337,7 @@
         "x": 18,
         "y": 6
       },
-      "id": 34,
+      "id": 35,
       "interval": null,
       "links": [],
       "mappingType": 1,
@@ -2337,7 +2416,7 @@
         "x": 20,
         "y": 6
       },
-      "id": 35,
+      "id": 36,
       "interval": null,
       "links": [],
       "mappingType": 1,
@@ -2416,7 +2495,7 @@
         "x": 22,
         "y": 6
       },
-      "id": 36,
+      "id": 37,
       "interval": null,
       "links": [],
       "mappingType": 1,
@@ -2495,7 +2574,7 @@
         "x": 16,
         "y": 8
       },
-      "id": 37,
+      "id": 38,
       "interval": null,
       "links": [],
       "mappingType": 1,
@@ -2559,7 +2638,7 @@
         "x": 8,
         "y": 9
       },
-      "id": 38,
+      "id": 39,
       "links": [],
       "mode": "markdown",
       "title": "Infrastructure",
@@ -2588,7 +2667,7 @@
         "x": 8,
         "y": 10
       },
-      "id": 39,
+      "id": 40,
       "interval": null,
       "links": [],
       "mappingType": 1,
@@ -2667,7 +2746,7 @@
         "x": 10,
         "y": 10
       },
-      "id": 40,
+      "id": 41,
       "interval": null,
       "links": [],
       "mappingType": 1,
@@ -2746,7 +2825,7 @@
         "x": 12,
         "y": 10
       },
-      "id": 41,
+      "id": 42,
       "interval": null,
       "links": [],
       "mappingType": 1,
@@ -2825,7 +2904,7 @@
         "x": 14,
         "y": 10
       },
-      "id": 42,
+      "id": 43,
       "interval": null,
       "links": [],
       "mappingType": 1,