Skip to content

Commit

Permalink
Updates to dashboard generator, esp. for hadoop3
Browse files Browse the repository at this point in the history
  • Loading branch information
@GilHoggarth
GilHoggarth committed Nov 3, 2021
1 parent 4c82225 commit d0520f6
Show file tree
Hide file tree
Showing 4 changed files with 113 additions and 49 deletions.
15 changes: 8 additions & 7 deletions monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
# template files
header = 'templates/header'
panelHeader = 'templates/panelHeader'
panelBanner = 'templates/panelBanner'
panelTitle = 'templates/panelTitle'
panelSingle = 'templates/panelSingle'
panelSingleHadoopUsed = 'templates/panelSingleHadoopUsed'
Expand Down Expand Up @@ -137,14 +138,14 @@ def replace_output_single(outHandle, **kwargs):
expr = 'hdfs_used_percent{instance=\\"${HDFS_EXPORTER}\\"}'
templateCode = templateCode.replace('<expr>', expr)
elif kwargs['job'] == 'hadoop3':
if kwargs['title'] == 'Nodes':
expr = 'hadoop_hdfs_namenode_nninfo_live_nodes_count{instance=\\"${HDFS3_EXPORTER}\\"} - hadoop_hdfs_namenode_nninfo_dead_nodes_count{instance=\\"${HDFS3_EXPORTER}\\"}'
if kwargs['title'] == 'Dead Nodes':
expr = 'hadoop_hdfs_namenode_nninfo_dead_nodes_count{instance=\\"${HDFS3_EXPORTER}\\"}'
templateCode = templateCode.replace('<expr>', expr)
elif kwargs['title'] == 'Under-rep':
expr = 'hadoop_hdfs_namenode_fsname_system_under_replicated_blocks{instance=\\"${HDFS3_EXPORTER}\\"}'
templateCode = templateCode.replace('<expr>', expr)
elif kwargs['title'] == 'Used':
expr = '100 - hadoop_hdfs_namenode_nninfo_percent_remaining{instance=\\"${HDFS3_EXPORTER}\\"}'
expr = 'round(100 - hadoop_hdfs_namenode_nninfo_percent_remaining{instance=\\"${HDFS3_EXPORTER}\\"})'
templateCode = templateCode.replace('<expr>', expr)

# add last comma if not last panel
Expand Down Expand Up @@ -172,7 +173,7 @@ def main():
# output single panels -----

## storage title bar
replace_output_title(outHandle, pnl=panelTitle, job='hadoop', title='Storage', h=2, w=24, x=0, y=0)
replace_output_title(outHandle, pnl=panelBanner, job='hadoop', title='Storage', h=2, w=24, x=0, y=0)
## hadoop 0.20
replace_output_title(outHandle, pnl=panelTitle, job='hadoop', title='Hadoop 0.20', h=1, w=8, x=0, y=2)
replace_output_single(outHandle, pnl=panelStat, job='hadoop', title='Up', h=2, w=2, x=0, y=3)
Expand All @@ -192,14 +193,14 @@ def main():
replace_output_title(outHandle, pnl=panelTitle, job='hadoop3', title='Hadoop 3', h=1, w=8, x=16, y=2)
replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='Up', h=2, w=2, x=16, y=3)
replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='CPU', h=2, w=2, x=18, y=3)
replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='Nodes', h=2, w=2, x=20, y=3)
replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='Dead Nodes', h=2, w=2, x=20, y=3)
replace_output_single(outHandle, pnl=panelStatHadoopUsed, job='hadoop3', title='Used', threshold2='85', threshold3='90', h=4, w=2, x=22, y=3)
replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='Dsk', h=2, w=2, x=16, y=5)
replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='Mem', threshold2='0.1', threshold3='1.1', h=2, w=2, x=18, y=5)
replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='Under-rep', textmode='value', h=2, w=2, x=20, y=5)

## wa systems title bar
replace_output_title(outHandle, pnl=panelTitle, job='infrastructure', title='WA Systems', h=2, w=24, x=0, y=6)
replace_output_title(outHandle, pnl=panelBanner, job='infrastructure', title='WA Systems', h=2, w=24, x=0, y=6)
## infrastructure
replace_output_title(outHandle, pnl=panelTitle, job='infrastructure', title='Infrastructure', h=1, w=8, x=0, y=8)
replace_output_single(outHandle, pnl=panelStat, job='infrastructure', title='Up', h=2, w=2, x=0, y=9)
Expand All @@ -221,7 +222,7 @@ def main():
replace_output_single(outHandle, pnl=panelStat, job='solr-query', title='Query', h=2, w=2, x=16, y=11)

## wa services title bar
replace_output_title(outHandle, pnl=panelTitle, job='wa_services', title='WA Services', h=2, w=24, x=0, y=12)
replace_output_title(outHandle, pnl=panelBanner, job='wa_services', title='WA Services', h=2, w=24, x=0, y=12)
# ingest & metadata
replace_output_title(outHandle, pnl=panelTitle, job='ingest_metadata', title='Ingest & Metadata', h=1, w=8, x=0, y=14)
replace_output_single(outHandle, pnl=panelStat, job='ingest_metadata', title='Up', h=2, w=2, x=0, y=15)
Expand Down
21 changes: 21 additions & 0 deletions monitor/grafana/provisioning/_dashboard_generator/templates/panelBanner
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"type": "text",
"gridPos": {
"h": <h>,
"w": <w>,
"x": <x>,
"y": <y>
},
"id": <id>,
"options": {
"mode": "markdown",
"content": "# <title>"
},
"transparent": true,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"datasource": null,
"pluginVersion": "7.5.7"
},
63 changes: 42 additions & 21 deletions monitor/grafana/provisioning/dashboards/wa_status.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,26 @@
"id": 5,
"links": [],
"panels": [
{
"content": "",
{
"type": "text",
"gridPos": {
"h": 2,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"links": [],
"mode": "markdown",
"title": "Storage",
"type": "text"
"options": {
"mode": "markdown",
"content": "# Storage"
},
"transparent": true,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"datasource": null,
"pluginVersion": "7.5.7"
},
{
"content": "",
Expand Down Expand Up @@ -1120,7 +1127,7 @@
"pluginVersion": "7.1.3",
"targets": [
{
"expr": "hadoop_hdfs_namenode_nninfo_live_nodes_count{instance=\"h3exporter.api.wa.bl.uk:80\"} - hadoop_hdfs_namenode_nninfo_dead_nodes_count{instance=\"h3exporter.api.wa.bl.uk:80\"}",
"expr": "hadoop_hdfs_namenode_nninfo_dead_nodes_count{instance=\"h3exporter.api.wa.bl.uk:80\"}",
"legendFormat": "",
"interval": "",
"format": "time_series",
Expand All @@ -1130,7 +1137,7 @@
}
],
"type": "stat",
"title": "Nodes",
"title": "Dead Nodes",
"cacheTimeout": null,
"interval": null,
"datasource": null
Expand All @@ -1147,7 +1154,7 @@
"maxDataPoints": 100,
"targets": [
{
"expr": "100 - hadoop_hdfs_namenode_nninfo_percent_remaining{instance=\"h3exporter.api.wa.bl.uk:80\"}",
"expr": "round(100 - hadoop_hdfs_namenode_nninfo_percent_remaining{instance=\"h3exporter.api.wa.bl.uk:80\"})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
Expand Down Expand Up @@ -1436,19 +1443,26 @@
"interval": null,
"datasource": null
},
{
"content": "",
{
"type": "text",
"gridPos": {
"h": 2,
"w": 24,
"x": 0,
"y": 6
},
"id": 23,
"links": [],
"mode": "markdown",
"title": "WA Systems",
"type": "text"
"options": {
"mode": "markdown",
"content": "# WA Systems"
},
"transparent": true,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"datasource": null,
"pluginVersion": "7.5.7"
},
{
"content": "",
Expand Down Expand Up @@ -2480,19 +2494,26 @@
"interval": null,
"datasource": null
},
{
"content": "",
{
"type": "text",
"gridPos": {
"h": 2,
"w": 24,
"x": 0,
"y": 12
},
"id": 40,
"links": [],
"mode": "markdown",
"title": "WA Services",
"type": "text"
"options": {
"mode": "markdown",
"content": "# WA Services"
},
"transparent": true,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"datasource": null,
"pluginVersion": "7.5.7"
},
{
"content": "",
Expand Down
63 changes: 42 additions & 21 deletions monitor/grafana/provisioning/dashboards/wa_status.json-template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,26 @@
"id": 5,
"links": [],
"panels": [
{
"content": "",
{
"type": "text",
"gridPos": {
"h": 2,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"links": [],
"mode": "markdown",
"title": "Storage",
"type": "text"
"options": {
"mode": "markdown",
"content": "# Storage"
},
"transparent": true,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"datasource": null,
"pluginVersion": "7.5.7"
},
{
"content": "",
Expand Down Expand Up @@ -1120,7 +1127,7 @@
"pluginVersion": "7.1.3",
"targets": [
{
"expr": "hadoop_hdfs_namenode_nninfo_live_nodes_count{instance=\"${HDFS3_EXPORTER}\"} - hadoop_hdfs_namenode_nninfo_dead_nodes_count{instance=\"${HDFS3_EXPORTER}\"}",
"expr": "hadoop_hdfs_namenode_nninfo_dead_nodes_count{instance=\"${HDFS3_EXPORTER}\"}",
"legendFormat": "",
"interval": "",
"format": "time_series",
Expand All @@ -1130,7 +1137,7 @@
}
],
"type": "stat",
"title": "Nodes",
"title": "Dead Nodes",
"cacheTimeout": null,
"interval": null,
"datasource": null
Expand All @@ -1147,7 +1154,7 @@
"maxDataPoints": 100,
"targets": [
{
"expr": "100 - hadoop_hdfs_namenode_nninfo_percent_remaining{instance=\"${HDFS3_EXPORTER}\"}",
"expr": "round(100 - hadoop_hdfs_namenode_nninfo_percent_remaining{instance=\"${HDFS3_EXPORTER}\"})",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
Expand Down Expand Up @@ -1436,19 +1443,26 @@
"interval": null,
"datasource": null
},
{
"content": "",
{
"type": "text",
"gridPos": {
"h": 2,
"w": 24,
"x": 0,
"y": 6
},
"id": 23,
"links": [],
"mode": "markdown",
"title": "WA Systems",
"type": "text"
"options": {
"mode": "markdown",
"content": "# WA Systems"
},
"transparent": true,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"datasource": null,
"pluginVersion": "7.5.7"
},
{
"content": "",
Expand Down Expand Up @@ -2480,19 +2494,26 @@
"interval": null,
"datasource": null
},
{
"content": "",
{
"type": "text",
"gridPos": {
"h": 2,
"w": 24,
"x": 0,
"y": 12
},
"id": 40,
"links": [],
"mode": "markdown",
"title": "WA Services",
"type": "text"
"options": {
"mode": "markdown",
"content": "# WA Services"
},
"transparent": true,
"fieldConfig": {
"defaults": {},
"overrides": []
},
"datasource": null,
"pluginVersion": "7.5.7"
},
{
"content": "",
Expand Down

0 comments on commit d0520f6

Please sign in to comment.