From 734aa1ed3a24b6e88e17522cbbab179e5e430c0a Mon Sep 17 00:00:00 2001 From: Gil Hoggarth Date: Tue, 31 Jan 2023 16:19:23 +0000 Subject: [PATCH 1/2] Removed section banners, to reduce used space --- .gitignore | 4 ++++ .../_dashboard_generator/generate_wa-status_dashboard.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index f6b5c6a..75a6bbb 100644 --- a/.gitignore +++ b/.gitignore @@ -5,11 +5,15 @@ venv/ ENV/ +# files *.pyc *out *log *pid *lock +.swp + +# generated content stat-pusher/dev[0-9]-*-[0-9]* stat-pusher/beta[0-9]-*-[0-9]* stat-pusher/prod[0-9]-*-[0-9]* diff --git a/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py b/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py index a708b06..160fc02 100755 --- a/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py +++ b/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py @@ -18,7 +18,7 @@ # template files header = 'templates/header' panelHeader = 'templates/panelHeader' -panelBanner = 'templates/panelBanner' +#panelBanner = 'templates/panelBanner' panelTitle = 'templates/panelTitle' panelSingle = 'templates/panelSingle' panelSingleHadoopUsed = 'templates/panelSingleHadoopUsed' @@ -176,7 +176,7 @@ def main(): # output single panels ----- ## storage title bar - replace_output_title(outHandle, pnl=panelBanner, job='hadoop', title='Storage', h=2, w=24, x=0, y=0) +# replace_output_title(outHandle, pnl=panelBanner, job='hadoop', title='Storage', h=2, w=24, x=0, y=0) ## hadoop 0.20 replace_output_title(outHandle, pnl=panelTitle, job='hadoop', title='Hadoop 0.20', h=1, w=8, x=0, y=2) replace_output_single(outHandle, pnl=panelStat, job='hadoop', title='Up', h=2, w=2, x=0, y=3) @@ -203,7 +203,7 @@ def main(): replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='Under-rep', textmode='value', h=2, w=2, x=20, y=5) ## wa systems title bar - replace_output_title(outHandle, pnl=panelBanner, job='infrastructure', title='WA Systems', h=2, w=24, x=0, y=6) +# replace_output_title(outHandle, pnl=panelBanner, job='infrastructure', title='WA Systems', h=2, w=24, x=0, y=6) ## infrastructure replace_output_title(outHandle, pnl=panelTitle, job='infrastructure', title='Infrastructure', h=1, w=8, x=0, y=8) replace_output_single(outHandle, pnl=panelStat, job='infrastructure', title='Up', h=2, w=2, x=0, y=9) @@ -226,7 +226,7 @@ def main(): replace_output_single(outHandle, pnl=panelStat, job='solr-query', title='Query', h=2, w=2, x=16, y=11) ## wa services title bar - replace_output_title(outHandle, pnl=panelBanner, job='wa_services', title='WA Services', h=2, w=24, x=0, y=12) +# replace_output_title(outHandle, pnl=panelBanner, job='wa_services', title='WA Services', h=2, w=24, x=0, y=12) # ingest & metadata replace_output_title(outHandle, pnl=panelTitle, job='ingest_metadata', title='Ingest & Metadata', h=1, w=8, x=0, y=14) replace_output_single(outHandle, pnl=panelStat, job='ingest_metadata', title='Up', h=2, w=2, x=0, y=15) From aacd2702f5143675d6958a41f26598ae82a7dc0f Mon Sep 17 00:00:00 2001 From: Gil Hoggarth Date: Wed, 1 Feb 2023 16:32:18 +0000 Subject: [PATCH 2/2] Removed section titles/general space to status dashboard --- .../generate_wa-status_dashboard.py | 261 -------------- .../generate_wa-status_dashboard_g7.py | 187 ++++++++++ .../_dashboard_generator/modules7/replace.py | 126 +++++++ .../_dashboard_generator/modules7/rw.py | 9 + .../{templates => templates7}/footer | 0 .../{templates => templates7}/header | 0 .../{templates => templates7}/panelBanner | 0 .../{templates => templates7}/panelFooter | 0 .../{templates => templates7}/panelHeader | 0 .../{templates => templates7}/panelSingle | 0 .../panelSingleHadoopUsed | 0 .../templates7/panelSpacer | 23 ++ .../{templates => templates7}/panelStat | 0 .../panelStatHadoopUsed | 0 .../{templates => templates7}/panelTitle | 0 .../dashboards/wa_status.json-template | 331 +++++++++--------- 16 files changed, 502 insertions(+), 435 deletions(-) delete mode 100755 monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py create mode 100755 monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard_g7.py create mode 100755 monitor/grafana/provisioning/_dashboard_generator/modules7/replace.py create mode 100755 monitor/grafana/provisioning/_dashboard_generator/modules7/rw.py rename monitor/grafana/provisioning/_dashboard_generator/{templates => templates7}/footer (100%) rename monitor/grafana/provisioning/_dashboard_generator/{templates => templates7}/header (100%) rename monitor/grafana/provisioning/_dashboard_generator/{templates => templates7}/panelBanner (100%) rename monitor/grafana/provisioning/_dashboard_generator/{templates => templates7}/panelFooter (100%) rename monitor/grafana/provisioning/_dashboard_generator/{templates => templates7}/panelHeader (100%) rename monitor/grafana/provisioning/_dashboard_generator/{templates => templates7}/panelSingle (100%) rename monitor/grafana/provisioning/_dashboard_generator/{templates => templates7}/panelSingleHadoopUsed (100%) create mode 100644 monitor/grafana/provisioning/_dashboard_generator/templates7/panelSpacer rename monitor/grafana/provisioning/_dashboard_generator/{templates => templates7}/panelStat (100%) rename monitor/grafana/provisioning/_dashboard_generator/{templates => templates7}/panelStatHadoopUsed (100%) rename monitor/grafana/provisioning/_dashboard_generator/{templates => templates7}/panelTitle (100%) diff --git a/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py b/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py deleted file mode 100755 index 160fc02..0000000 --- a/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python3.6 - -# Grafana seems unable to control the position of panels so that the addition -# of a new panel knocks others out of place. Plus, the repeat functions seems -# not to work. Consequently, this 'simple code' is to create the wa_status -# dashboard with easy repetition of panels and controlled positioning. - -import re -import datetime - -# output filename -outFile = 'wa_status.json-template' - -# dashboard prefixes -dashTitle = 'WA Status' -dashUid = 'wast' - -# template files -header = 'templates/header' -panelHeader = 'templates/panelHeader' -#panelBanner = 'templates/panelBanner' -panelTitle = 'templates/panelTitle' -panelSingle = 'templates/panelSingle' -panelSingleHadoopUsed = 'templates/panelSingleHadoopUsed' -panelStat = 'templates/panelStat' -panelStatHadoopUsed = 'templates/panelStatHadoopUsed' -panelFooter = 'templates/panelFooter' -footer = 'templates/footer' - -# panel ID, increments on usage so all unique in panels -ID = 0 - -# functions ------------------------------ -def nextID(): - global ID - ID=ID + 1 - return str(ID) - -def read_template(**kwargs): - tplC = '' - with open(kwargs['pnl'], 'r') as tC: - tplC = tC.read() - return tplC - -def output(outHandle, **kwargs): - templateCode = read_template(pnl=kwargs['pnl']) - outHandle.write(templateCode) - -def replace_output_title(outHandle, **kwargs): - templateCode = read_template(pnl=kwargs['pnl']) - templateCode = templateCode.replace('', kwargs['title']) - templateCode = templateCode.replace('<id>', nextID()) - templateCode = templateCode.replace('<h>', str(kwargs['h'])) - templateCode = templateCode.replace('<w>', str(kwargs['w'])) - templateCode = templateCode.replace('<x>', str(kwargs['x'])) - templateCode = templateCode.replace('<y>', str(kwargs['y'])) - outHandle.write(templateCode) - -def replace_output_single(outHandle, **kwargs): - templateCode = read_template(pnl=kwargs['pnl']) - templateCode = templateCode.replace('<title>', kwargs['title']) - templateCode = templateCode.replace('<id>', nextID()) - templateCode = templateCode.replace('<h>', str(kwargs['h'])) - templateCode = templateCode.replace('<w>', str(kwargs['w'])) - templateCode = templateCode.replace('<x>', str(kwargs['x'])) - templateCode = templateCode.replace('<y>', str(kwargs['y'])) - - # specific thresholds replacement - if kwargs['pnl'] == panelStat or kwargs['pnl'] == panelStatHadoopUsed: - if 'threshold1' in kwargs: - templateCode = templateCode.replace('<threshold1>', kwargs['threshold1']) - else: - templateCode = templateCode.replace('<threshold1>', 'null') - if 'threshold2' in kwargs: - templateCode = templateCode.replace('<threshold2>', kwargs['threshold2']) - else: - templateCode = templateCode.replace('<threshold2>', '0.1') - if 'threshold3' in kwargs: - templateCode = templateCode.replace('<threshold3>', kwargs['threshold3']) - else: - templateCode = templateCode.replace('<threshold3>', '1') - if 'textmode' in kwargs: - templateCode = templateCode.replace('<textmode>', kwargs['textmode']) - else: - templateCode = templateCode.replace('<textmode>', 'none') - else: - if 'thresholds' in kwargs: - templateCode = templateCode.replace('<thresholds>', kwargs['thresholds']) - else: - templateCode = templateCode.replace('<thresholds>', '0.1,1') # default threshhold - # specific colours replacement - if 'colour1' in kwargs: - templateCode = templateCode.replace('<colour1>', kwargs['colour1']) - else: - templateCode = templateCode.replace('<colour1>', '#299C46') # default panel okay colour - if 'colour2' in kwargs: - templateCode = templateCode.replace('<colour2>', kwargs['colour2']) - else: - templateCode = templateCode.replace('<colour2>', '#ED8027') # default panel warning colour - if 'colour3' in kwargs: - templateCode = templateCode.replace('<colour3>', kwargs['colour3']) - else: - templateCode = templateCode.replace('<colour3>', '#D44A3A') # default panel problem colour - - # add expr - if kwargs['title'] == 'Up': - expr = 'sum(1 - up{job=\\"' + kwargs['job'] + '\\"})' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['title'] == 'CPU': - expr = 'count(sum(irate(node_cpu_seconds_total{job=\\"' + kwargs['job'] + '\\",mode=\\"idle\\"}[5m]) < 0.1) by (instance)) OR vector(0)' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['title'] == 'Dsk': - expr = 'count((node_filesystem_avail_bytes{job=\\"' + kwargs['job'] + '\\",fstype!~\\"tmpfs|rootfs|cifs\\"} / node_filesystem_size_bytes{job=\\"' + kwargs['job'] + '\\",fstype!~\\"tmpfs|rootfs|cifs\\"}) < 0.04) OR vector(0)' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['title'] == 'Mem': - expr = 'count(sum(node_memory_MemFree_bytes{job=\\"' + kwargs['job'] + '\\"} + node_memory_Buffers_bytes{job=\\"' + kwargs['job'] + '\\"} + node_memory_Cached_bytes{job=\\"' + kwargs['job'] + '\\"}) by (instance) / sum(node_memory_MemTotal_bytes{job=\\"' + kwargs['job'] + '\\"}) by (instance) < 0.05) OR vector(0)' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['title'] == 'UTR': - expr = 'count(uptimerobot_monitor_up==0) OR vector(0)' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['title'] == 'WWW' or kwargs['title'] == 'Query': - expr = 'count(probe_http_status_code{job=\\"' + kwargs['job'] + '\\"} != 200) OR vector(0)' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['title'] == 'trackdb refresh': - expr = '(time() - trackdb_refresh_timestamp) / (60*60)' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['title'] == 'trackdb numFound': - expr = 'sum(trackdb_numFound - (trackdb_numFound offset 1d))' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['job'] == 'hadoop': - if kwargs['title'] == 'Nodes': - expr = 'hdfs_node_count{status=\\"dead\\",instance=\\"${HDFS_EXPORTER}\\"}' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['title'] == 'Under-rep': - expr = 'hdfs_under_replicated_block_count{instance=\\"${HDFS_EXPORTER}\\"}' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['title'] == 'Used': - expr = 'hdfs_used_percent{instance=\\"${HDFS_EXPORTER}\\"}' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['job'] == 'hadoop3': - if kwargs['title'] == 'Dead Nodes': - expr = 'hadoop_hdfs_namenode_nninfo_dead_nodes_count{instance=\\"${HDFS3_EXPORTER}\\"}' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['title'] == 'Under-rep': - expr = 'hadoop_hdfs_namenode_fsname_system_under_replicated_blocks{instance=\\"${HDFS3_EXPORTER}\\"}' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['title'] == 'Used': - expr = 'round(100 - hadoop_hdfs_namenode_nninfo_percent_remaining{instance=\\"${HDFS3_EXPORTER}\\"})' - templateCode = templateCode.replace('<expr>', expr) - elif kwargs['title'] == 'LDLs': - expr = 'sum(last_over_time(recent_connections{job=\\"ldl_rr\\",instance=~\\"DLS-.+\\"}[30m]))' - templateCode = templateCode.replace('<expr>', expr) - - # add last comma if not last panel - if 'lastPanel' not in kwargs: - templateCode = re.sub(r'}$', '},', templateCode) - outHandle.write(templateCode) - -def replace_output_footer(outHandle, **kwargs): - templateCode = read_template(pnl=kwargs['pnl']) - templateCode = templateCode.replace('<title>', kwargs['title']) - templateCode = templateCode.replace('<uid>', kwargs['uid']) - outHandle.write(templateCode) - -# main ----------------------------------- -def main(): - # get output handle for output file - outHandle = open(outFile, 'w') - - # write page header template to output handle - output(outHandle, pnl = header) - - # write panel header to output handle - output(outHandle, pnl = panelHeader) - - # output single panels ----- - - ## storage title bar -# replace_output_title(outHandle, pnl=panelBanner, job='hadoop', title='Storage', h=2, w=24, x=0, y=0) - ## hadoop 0.20 - replace_output_title(outHandle, pnl=panelTitle, job='hadoop', title='Hadoop 0.20', h=1, w=8, x=0, y=2) - replace_output_single(outHandle, pnl=panelStat, job='hadoop', title='Up', h=2, w=2, x=0, y=3) - replace_output_single(outHandle, pnl=panelStat, job='hadoop', title='CPU', h=2, w=2, x=2, y=3) - replace_output_single(outHandle, pnl=panelStat, job='hadoop', title='Nodes', h=2, w=2, x=4, y=3) - replace_output_single(outHandle, pnl=panelStatHadoopUsed, job='hadoop', title='Used', threshold2='85', threshold3='90', h=4, w=2, x=6, y=3) - replace_output_single(outHandle, pnl=panelStat, job='hadoop', title='Dsk', h=2, w=2, x=0, y=5) - replace_output_single(outHandle, pnl=panelStat, job='hadoop', title='Mem', threshold2='0.1', threshold3='1.1', h=2, w=2, x=2, y=5) - replace_output_single(outHandle, pnl=panelStat, job='hadoop', title='Under-rep', h=2, w=2, x=4, y=5) - # gluster - replace_output_title(outHandle, pnl=panelTitle, job='gluster', title='Gluster', h=1, w=8, x=8, y=2) - replace_output_single(outHandle, pnl=panelStat, job='gluster', title='Up', h=2, w=2, x=8, y=3) - replace_output_single(outHandle, pnl=panelStat, job='gluster', title='CPU', h=2, w=2, x=10, y=3) - replace_output_single(outHandle, pnl=panelStat, job='gluster', title='Dsk', h=2, w=2, x=12, y=3) - replace_output_single(outHandle, pnl=panelStat, job='gluster', title='Mem', h=2, w=2, x=14, y=3) - ## hadoop 3 - replace_output_title(outHandle, pnl=panelTitle, job='hadoop3', title='Hadoop 3', h=1, w=8, x=16, y=2) - replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='Up', h=2, w=2, x=16, y=3) - replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='CPU', h=2, w=2, x=18, y=3) - replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='Dead Nodes', h=2, w=2, x=20, y=3) - replace_output_single(outHandle, pnl=panelStatHadoopUsed, job='hadoop3', title='Used', threshold2='85', threshold3='90', h=4, w=2, x=22, y=3) - replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='Dsk', h=2, w=2, x=16, y=5) - replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='Mem', threshold2='0.1', threshold3='1.1', h=2, w=2, x=18, y=5) - replace_output_single(outHandle, pnl=panelStat, job='hadoop3', title='Under-rep', textmode='value', h=2, w=2, x=20, y=5) - - ## wa systems title bar -# replace_output_title(outHandle, pnl=panelBanner, job='infrastructure', title='WA Systems', h=2, w=24, x=0, y=6) - ## infrastructure - replace_output_title(outHandle, pnl=panelTitle, job='infrastructure', title='Infrastructure', h=1, w=8, x=0, y=8) - replace_output_single(outHandle, pnl=panelStat, job='infrastructure', title='Up', h=2, w=2, x=0, y=9) - replace_output_single(outHandle, pnl=panelStat, job='infrastructure', title='CPU', h=2, w=2, x=2, y=9) - replace_output_single(outHandle, pnl=panelStat, job='infrastructure', title='Dsk', h=2, w=2, x=4, y=9) - replace_output_single(outHandle, pnl=panelStat, job = 'infrastructure', title = 'Mem', h=2, w=2, x=6, y=9) - replace_output_single(outHandle, pnl=panelStat, job='infrastructure', title='LDLs', h=2, w=2, x=0, y=11, textmode='value', colour1='#D44A3A', colour3='#299C46', threshold2='8.5', threshold3='9.5') - # general services - replace_output_title(outHandle, pnl=panelTitle, job='services', title='Services', h=1, w=8, x=8, y=8) - replace_output_single(outHandle, pnl=panelStat, job='services', title='Up', h=2, w=2, x=8, y=9) - replace_output_single(outHandle, pnl=panelStat, job='services', title='CPU', h=2, w=2, x=10, y=9) - replace_output_single(outHandle, pnl=panelStat, job='services', title='Dsk', h=2, w=2, x=12, y=9) - replace_output_single(outHandle, pnl=panelStat, job='services', title='Mem', h=2, w=2, x=14, y=9) - # solr servers - replace_output_title(outHandle, pnl=panelTitle, job='solr', title='Solr', h=1, w=8, x=16, y=8) - replace_output_single(outHandle, pnl=panelStat, job='solr', title='Up', h=2, w=2, x=16, y=9) - replace_output_single(outHandle, pnl=panelStat, job='solr', title='CPU', h=2, w=2, x=18, y=9) - replace_output_single(outHandle, pnl=panelStat, job='solr', title='Dsk', h=2, w=2, x=20, y=9) - replace_output_single(outHandle, pnl=panelStat, job='solr', title='Mem', h=2, w=2, x=22, y=9) - replace_output_single(outHandle, pnl=panelStat, job='solr-query', title='Query', h=2, w=2, x=16, y=11) - - ## wa services title bar -# replace_output_title(outHandle, pnl=panelBanner, job='wa_services', title='WA Services', h=2, w=24, x=0, y=12) - # ingest & metadata - replace_output_title(outHandle, pnl=panelTitle, job='ingest_metadata', title='Ingest & Metadata', h=1, w=8, x=0, y=14) - replace_output_single(outHandle, pnl=panelStat, job='ingest_metadata', title='Up', h=2, w=2, x=0, y=15) - replace_output_single(outHandle, pnl=panelStat, job='ingest_metadata', title='CPU', threshold2='0.1', threshold3='1.1', colour2='#ba43a9', h=2, w=2, x=2, y=15) - replace_output_single(outHandle, pnl=panelStat, job='ingest_metadata', title='Dsk', h=2, w=2, x=4, y=15) - replace_output_single(outHandle, pnl=panelStat, job='ingest_metadata', title='Mem', h=2, w=2, x=6, y=15) - replace_output_single(outHandle, pnl=panelStat, job='im-access-http', title='WWW', h=2, w=2, x=0, y=17) - #trackdb - replace_output_title(outHandle, pnl=panelTitle, job='trackdb', title='TrackDB', h=1, w=8, x=8, y=14) - replace_output_single(outHandle, pnl=panelStat, job='trackdb', title='trackdb refresh', textmode='value', threshold2='22', threshold3='26', h=2, w=4, x=8, y=15) - replace_output_single(outHandle, pnl=panelStat, job='trackdb', title='trackdb numFound', textmode='value', threshold2='10', threshold3='100', colour1='#D44A3A', colour3='#299C46', h=2, w=4, x=12, y=15) - # discovery & access - replace_output_title(outHandle, pnl=panelTitle, job='discovery_access', title='Discovery & Access', h=1, w=8, x=16, y=14) - replace_output_single(outHandle, pnl=panelStat, job='discovery_access', title='Up', h=2, w=2, x=16, y=15) - replace_output_single(outHandle, pnl=panelStat, job='discovery_access', title='CPU', h=2, w=2, x=18, y=15) - replace_output_single(outHandle, pnl=panelStat, job='discovery_access', title='Dsk', h=2, w=2, x=20, y=15) - replace_output_single(outHandle, pnl=panelStat, job='discovery_access', title='Mem', threshold2='0.1', threshold3='1.1', colour2='#ba43a9', h=2, w=2, x=22, y=15) - replace_output_single(outHandle, pnl=panelStat, job='discovery_access', title='UTR', threshold2='0.1', threshold3='2.1', h=2, w=2, x=16, y=17) - #### tag last panel as 'lastPanel=True' so that the template ',' is removed (to make json valid) - replace_output_single(outHandle, pnl=panelStat, job='da-access-http', title='WWW', threshold2='0.1', threshold3='1.1', h=2, w=2, x=18, y=17, lastPanel=True) - - - # amend dashboard values - output(outHandle, pnl = panelFooter) - date = datetime.datetime.now() - formattedDate = date.strftime("%Y%m%d%H%M%S") - replace_output_footer(outHandle, pnl=footer, title=dashTitle, uid=dashUid) - - outHandle.close() - -if __name__ == '__main__': - main() diff --git a/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard_g7.py b/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard_g7.py new file mode 100755 index 0000000..0e5d978 --- /dev/null +++ b/monitor/grafana/provisioning/_dashboard_generator/generate_wa-status_dashboard_g7.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# 'Simple code' is to create the wa_status dashboard with easy repetition of panels and controlled positioning. +# Script name includes 'g7' as targetted for grafana v7 (panels, visualisations seem to get replaced over time; e.g., previous v5 used singleStat) + +# modules for 'g7' +from modules7 import rw +from modules7 import replace + +# output filename +outFile = 'wa_status.json-template' +# dashboard prefixes +dashTitle = 'WA Status' +dashUid = 'wast' +# template files for 'g7' +header = 'templates7/header' +panelHeader = 'templates7/panelHeader' +panelBanner = 'templates7/panelBanner' +panelTitle = 'templates7/panelTitle' +panelStat = 'templates7/panelStat' +panelStatHadoopUsed = 'templates7/panelStatHadoopUsed' +panelSpace = 'templates7/panelSpacer' +panelFooter = 'templates7/panelFooter' +footer = 'templates7/footer' +# hwxy values +HEIGHT = 2 +WIDTH = 2 +FIRSTCOL = 0 +titleHeight = 1 +blockWidth = 8 + +# main ----------------------------------- +def script(): + oH = open(outFile, 'w') # get output handle (oH) for output file + rw.output(oH, pnl=header) # write page header template to output handle + rw.output(oH, pnl=panelHeader) # write panel header to output handle + + # output single panels ----- + # hwxy: As grafana panel positioning shrinks to earliest, top left position, hwxy defined here isn't necessarily implemented + # Consequently, below hxwy vars are used to try to set panel positions relative to previous panels. + # hw is used solely for panelStat height and width, as these are key values and other panels vary too much. + # Comment at line end of each panel represents what the xy values are expected to be following the intention of the vars (x.y). + # These can be checked with actual values in grafana, via panel Inspect > Panel JSON + + # first 'row' of sections ----- + blockRow = 0 + + ## hadoop 0.20 + xpos = FIRSTCOL # panel x position for hadoop 0.20, 0 + ypos = blockRow # panel y position for hadoop 0.20, 0 + blockCol = xpos # set block beginning column + replace.title(oH, pnl=panelTitle, title='Hadoop 0.20', h=titleHeight, w=blockWidth, x=xpos, y=ypos) # 0.0 + ypos += titleHeight # increment ypos by title height + for _title in ['Up', 'CPU', 'Nodes']: + replace.single(oH, pnl=panelStat, job='hadoop', title=_title, h=HEIGHT, w=WIDTH, x=xpos, y=ypos) # 0.1, 2.1, 4.1 + xpos += WIDTH # increment xpos by WIDTH + replace.single(oH, pnl=panelStatHadoopUsed, job='hadoop', title='Used', threshold2='85', threshold3='96', h=4, w=2, x=xpos, y=ypos) # 6.1 + xpos = blockCol # reset xpos to beginning of block + ypos += HEIGHT # increment ypos by HEIGHT + replace.single(oH, pnl=panelStat, job='hadoop', title='Dsk', h=HEIGHT, w=WIDTH, x=xpos, y=ypos) # 0.3 + xpos += WIDTH # increment xpos by WIDTH + replace.single(oH, pnl=panelStat, job='hadoop', title='Mem', threshold2='0.1', threshold3='1.1', h=HEIGHT, w=WIDTH, x=xpos, y=ypos) # 2.3 + xpos += WIDTH # increment xpos by WIDTH + replace.single(oH, pnl=panelStat, job='hadoop', title='Under-rep', h=HEIGHT, w=WIDTH, x=xpos, y=ypos) # 4.3 + + # gluster + xpos = blockWidth # panel x position for gluster, 8 + ypos = blockRow # panel x position for gluster, 0 + blockCol = xpos + replace.title(oH, pnl=panelTitle, title='Gluster', h=titleHeight, w=blockWidth, x=xpos, y=ypos) # 8.0 + ypos += titleHeight + for _title in ['Up', 'CPU', 'Dsk', 'Mem']: + replace.single(oH, pnl=panelStat, job='gluster', title=_title, h=HEIGHT, w=WIDTH, x=xpos, y=ypos) # 8.1, 10.1, 12.1, 14.1 + xpos += WIDTH + xpos = blockCol + ypos += HEIGHT + replace.space(oH, pnl=panelSpace, h=HEIGHT, w=blockWidth, x=xpos, y=ypos) # 8.3 + + ## hadoop 3 + xpos += blockWidth # panel x position for hadoop 3, blockWidth as previous spacer, 16 + ypos = blockRow # panel x position for hadoop 3, 0 + blockCol = xpos + replace.title(oH, pnl=panelTitle, title='Hadoop 3', h=titleHeight, w=blockWidth, x=xpos, y=ypos) # 16.0 + ypos += titleHeight + for _title in ['Up', 'CPU', 'Dead Nodes']: + replace.single(oH, pnl=panelStat, job='hadoop3', title=_title, h=HEIGHT, w=WIDTH, x=xpos, y=ypos) # 16.1, 18.1, 20.1 + xpos += WIDTH + replace.single(oH, pnl=panelStatHadoopUsed, job='hadoop3', title='Used', threshold2='85', threshold3='96', h=4, w=2, x=xpos, y=ypos) # 22.1 + xpos = blockCol + ypos += HEIGHT + replace.single(oH, pnl=panelStat, job='hadoop3', title='Dsk', h=HEIGHT, w=WIDTH, x=xpos, y=ypos) # 16.3 + xpos += WIDTH + replace.single(oH, pnl=panelStat, job='hadoop3', title='Mem', threshold2='0.1', threshold3='1.1', h=HEIGHT, w=WIDTH, x=xpos, y=ypos) # 18.3 + xpos += WIDTH + replace.single(oH, pnl=panelStat, job='hadoop3', title='Under-rep', h=HEIGHT, w=WIDTH, x=xpos, y=ypos) # 20.3 + + # second 'row' of sections ----- + blockRow = (titleHeight + HEIGHT + HEIGHT) # blockRow set to exceed above titles and panels, 5 + + ## infrastructure + xpos = FIRSTCOL # panel x position for infrastructure, 0 + ypos = blockRow # panel x position for infrastructure, 5 + blockCol = xpos + replace.title(oH, pnl=panelTitle, title='Infrastructure', h=titleHeight, w=blockWidth, x=xpos, y=ypos) + ypos += titleHeight + for _title in ['Up', 'CPU', 'Dsk', 'Mem']: + replace.single(oH, pnl=panelStat, job='infrastructure', title=_title, h=HEIGHT, w=WIDTH, x=xpos, y=ypos) + xpos += WIDTH + xpos = blockCol + ypos += HEIGHT + replace.single(oH, pnl=panelStat, job='infrastructure', title='LDLs', h=HEIGHT, w=WIDTH, x=xpos, y=ypos, textmode='value', colour1='#D44A3A', colour3='#299C46', threshold2='8.5', threshold3='9.5') + + # services + xpos = blockWidth # panel x position for services, 8 + ypos = blockRow # panel x position for services, 5 + blockCol = xpos + replace.title(oH, pnl=panelTitle, title='Services', h=titleHeight, w=blockWidth, x=xpos, y=ypos) + ypos += titleHeight + for _title in ['Up', 'CPU', 'Dsk', 'Mem']: + replace.single(oH, pnl=panelStat, job='services', title=_title, h=HEIGHT, w=WIDTH, x=xpos, y=ypos) + xpos += WIDTH + xpos = blockCol + ypos += HEIGHT + replace.space(oH, pnl=panelSpace, h=HEIGHT, w=blockWidth, x=xpos, y=ypos) + + # solr servers + xpos += blockWidth # panel x position for solr, 16 + ypos = blockRow # panel x position for solr, 5 + blockCol = xpos + replace.title(oH, pnl=panelTitle, title='Solr', h=titleHeight, w=blockWidth, x=xpos, y=ypos) + ypos += titleHeight + for _title in ['Up', 'CPU', 'Dsk', 'Mem']: + replace.single(oH, pnl=panelStat, job='solr', title=_title, h=HEIGHT, w=WIDTH, x=xpos, y=ypos) + xpos += WIDTH + xpos = blockCol + ypos += HEIGHT + replace.single(oH, pnl=panelStat, job='solr-query', title='Query', h=HEIGHT, w=WIDTH, x=xpos, y=ypos) + + # third 'row' of sections ----- + blockRow = ((titleHeight + HEIGHT + HEIGHT) *2) # blockRow set to exceed above titles and panels, 10 + + # ingest & metadata + xpos = FIRSTCOL # panel x position for ingest & metadata, 0 + ypos = blockRow # panel x position for ingest & metadata, 10 + blockCol = xpos + replace.title(oH, pnl=panelTitle, title='Ingest & Metadata', h=titleHeight, w=blockWidth, x=xpos, y=ypos) + ypos += titleHeight + for _title in ['Up', 'CPU', 'Dsk', 'Mem']: + replace.single(oH, pnl=panelStat, job='ingest_metadata', title=_title, h=HEIGHT, w=WIDTH, x=xpos, y=ypos) + xpos += WIDTH + xpos = blockCol + ypos += HEIGHT + replace.single(oH, pnl=panelStat, job='im-access-http', title='WWW', h=HEIGHT, w=WIDTH, x=xpos, y=ypos) + + #trackdb + xpos = blockWidth # panel x position for trackdb, 8 + ypos = blockRow # panel x position for trackdb, 10 + replace.title(oH, pnl=panelTitle, title='TrackDB', h=titleHeight, w=blockWidth, x=xpos, y=ypos) + replace.single(oH, pnl=panelStat, job='trackdb', title='Refresh', textmode='value', threshold2='22', threshold3='26', h=HEIGHT, w=(WIDTH*2), x=xpos, y=ypos) + xpos += (WIDTH * 2) + replace.single(oH, pnl=panelStat, job='trackdb', title='numFound', textmode='value', threshold2='10', threshold3='100', colour1='#D44A3A', colour3='#299C46', h=HEIGHT, w=(WIDTH*2), x=xpos, y=ypos) + + # discovery & access + xpos += (blockWidth / 2) # panel x position for discovery_access + ypos = blockRow # panel x position for discovery_access + blockCol = xpos + replace.title(oH, pnl=panelTitle, title='Discovery & Access', h=titleHeight, w=blockWidth, x=xpos, y=ypos) + ypos += titleHeight + for _title in ['Up', 'CPU', 'Dsk', 'Mem']: + replace.single(oH, pnl=panelStat, job='discovery_access', title=_title, h=HEIGHT, w=WIDTH, x=xpos, y=ypos) + xpos += WIDTH + xpos = blockCol + ypos += HEIGHT + replace.single(oH, pnl=panelStat, job='discovery_access', title='UTR', threshold2='0.1', threshold3='2.1', h=HEIGHT, w=WIDTH, x=xpos, y=ypos) + + #### tag last panel as 'lastPanel=True' so that the template ',' is removed (to make json valid) + xpos += WIDTH + replace.single(oH, pnl=panelStat, job='da-access-http', title='WWW', threshold2='0.1', threshold3='1.1', h=HEIGHT, w=WIDTH, x=xpos, y=ypos, lastPanel=True) + + + # amend dashboard values + rw.output(oH, pnl = panelFooter) + replace.footer(oH, pnl=footer, title=dashTitle, uid=dashUid) + + oH.close() + +if __name__ == '__main__': + script() diff --git a/monitor/grafana/provisioning/_dashboard_generator/modules7/replace.py b/monitor/grafana/provisioning/_dashboard_generator/modules7/replace.py new file mode 100755 index 0000000..4eadd0f --- /dev/null +++ b/monitor/grafana/provisioning/_dashboard_generator/modules7/replace.py @@ -0,0 +1,126 @@ +import re +from modules7 import rw + +# hacky copy of template variables and values +panelStat = 'templates7/panelStat' +panelStatHadoopUsed = 'templates7/panelStatHadoopUsed' + +# panel ID, increments on usage so all unique in panels +ID = 0 + + +# internal functions ----- +def _next_id(): + global ID + ID += 1 + return str(ID) + +# functions -------------- +def space(oH, **kwargs): + templateCode = rw.read_template(pnl=kwargs['pnl']) + templateCode = templateCode.replace('<id>', _next_id()) + templateCode = templateCode.replace('<h>', str(kwargs['h'])) + templateCode = templateCode.replace('<w>', str(kwargs['w'])) + templateCode = templateCode.replace('<x>', str(kwargs['x'])) + templateCode = templateCode.replace('<y>', str(kwargs['y'])) + oH.write(templateCode) + +def title(oH, **kwargs): + templateCode = rw.read_template(pnl=kwargs['pnl']) + templateCode = templateCode.replace('<title>', kwargs['title']) + templateCode = templateCode.replace('<id>', _next_id()) + templateCode = templateCode.replace('<h>', str(kwargs['h'])) + templateCode = templateCode.replace('<w>', str(kwargs['w'])) + templateCode = templateCode.replace('<x>', str(kwargs['x'])) + templateCode = templateCode.replace('<y>', str(kwargs['y'])) + oH.write(templateCode) + +def single(oH, **kwargs): + templateCode = rw.read_template(pnl=kwargs['pnl']) + templateCode = templateCode.replace('<title>', kwargs['title']) + templateCode = templateCode.replace('<id>', _next_id()) + templateCode = templateCode.replace('<h>', str(kwargs['h'])) + templateCode = templateCode.replace('<w>', str(kwargs['w'])) + templateCode = templateCode.replace('<x>', str(kwargs['x'])) + templateCode = templateCode.replace('<y>', str(kwargs['y'])) + + # specific thresholds replacement + if kwargs['pnl'] == panelStat or kwargs['pnl'] == panelStatHadoopUsed: + if 'threshold1' in kwargs: templateCode = templateCode.replace('<threshold1>', kwargs['threshold1']) + else: templateCode = templateCode.replace('<threshold1>', 'null') + if 'threshold2' in kwargs: templateCode = templateCode.replace('<threshold2>', kwargs['threshold2']) + else: templateCode = templateCode.replace('<threshold2>', '0.1') + if 'threshold3' in kwargs: templateCode = templateCode.replace('<threshold3>', kwargs['threshold3']) + else: templateCode = templateCode.replace('<threshold3>', '1') + if 'textmode' in kwargs: templateCode = templateCode.replace('<textmode>', kwargs['textmode']) + else: templateCode = templateCode.replace('<textmode>', 'none') + else: + if 'thresholds' in kwargs: templateCode = templateCode.replace('<thresholds>', kwargs['thresholds']) + else: templateCode = templateCode.replace('<thresholds>', '0.1,1') # default threshhold + # specific colours replacement + if 'colour1' in kwargs: templateCode = templateCode.replace('<colour1>', kwargs['colour1']) + else: templateCode = templateCode.replace('<colour1>', '#299C46') # default panel okay colour + if 'colour2' in kwargs: templateCode = templateCode.replace('<colour2>', kwargs['colour2']) + else: templateCode = templateCode.replace('<colour2>', '#ED8027') # default panel warning colour + if 'colour3' in kwargs: templateCode = templateCode.replace('<colour3>', kwargs['colour3']) + else: templateCode = templateCode.replace('<colour3>', '#D44A3A') # default panel problem colour + + # add expr + if kwargs['title'] == 'Up': + expr = 'sum(1 - up{job=\\"' + kwargs['job'] + '\\"})' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['title'] == 'CPU': + expr = 'count(sum(irate(node_cpu_seconds_total{job=\\"' + kwargs['job'] + '\\",mode=\\"idle\\"}[5m]) < 0.1) by (instance)) OR vector(0)' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['title'] == 'Dsk': + expr = 'count((node_filesystem_avail_bytes{job=\\"' + kwargs['job'] + '\\",fstype!~\\"tmpfs|rootfs|cifs\\"} / node_filesystem_size_bytes{job=\\"' + kwargs['job'] + '\\",fstype!~\\"tmpfs|rootfs|cifs\\"}) < 0.04) OR vector(0)' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['title'] == 'Mem': + expr = 'count(sum(node_memory_MemFree_bytes{job=\\"' + kwargs['job'] + '\\"} + node_memory_Buffers_bytes{job=\\"' + kwargs['job'] + '\\"} + node_memory_Cached_bytes{job=\\"' + kwargs['job'] + '\\"}) by (instance) / sum(node_memory_MemTotal_bytes{job=\\"' + kwargs['job'] + '\\"}) by (instance) < 0.05) OR vector(0)' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['title'] == 'UTR': + expr = 'count(uptimerobot_monitor_up==0) OR vector(0)' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['title'] == 'WWW' or kwargs['title'] == 'Query': + expr = 'count(probe_http_status_code{job=\\"' + kwargs['job'] + '\\"} != 200) OR vector(0)' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['title'] == 'Refresh': + expr = '(time() - trackdb_refresh_timestamp) / (60*60)' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['title'] == 'numFound': + expr = 'sum(trackdb_numFound - (trackdb_numFound offset 1d))' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['job'] == 'hadoop': + if kwargs['title'] == 'Nodes': + expr = 'hdfs_node_count{status=\\"dead\\",instance=\\"${HDFS_EXPORTER}\\"}' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['title'] == 'Under-rep': + expr = 'hdfs_under_replicated_block_count{instance=\\"${HDFS_EXPORTER}\\"}' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['title'] == 'Used': + expr = 'hdfs_used_percent{instance=\\"${HDFS_EXPORTER}\\"}' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['job'] == 'hadoop3': + if kwargs['title'] == 'Dead Nodes': + expr = 'hadoop_hdfs_namenode_nninfo_dead_nodes_count{instance=\\"${HDFS3_EXPORTER}\\"}' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['title'] == 'Under-rep': + expr = 'hadoop_hdfs_namenode_fsname_system_under_replicated_blocks{instance=\\"${HDFS3_EXPORTER}\\"}' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['title'] == 'Used': + expr = 'round(100 - hadoop_hdfs_namenode_nninfo_percent_remaining{instance=\\"${HDFS3_EXPORTER}\\"})' + templateCode = templateCode.replace('<expr>', expr) + elif kwargs['title'] == 'LDLs': + expr = 'sum(last_over_time(recent_connections{job=\\"ldl_rr\\",instance=~\\"DLS-.+\\"}[30m]))' + templateCode = templateCode.replace('<expr>', expr) + + # add last comma if not last panel + if 'lastPanel' not in kwargs: + templateCode = re.sub(r'}$', '},', templateCode) + oH.write(templateCode) + +def footer(oH, **kwargs): + templateCode = rw.read_template(pnl=kwargs['pnl']) + templateCode = templateCode.replace('<title>', kwargs['title']) + templateCode = templateCode.replace('<uid>', kwargs['uid']) + oH.write(templateCode) diff --git a/monitor/grafana/provisioning/_dashboard_generator/modules7/rw.py b/monitor/grafana/provisioning/_dashboard_generator/modules7/rw.py new file mode 100755 index 0000000..cb14274 --- /dev/null +++ b/monitor/grafana/provisioning/_dashboard_generator/modules7/rw.py @@ -0,0 +1,9 @@ +def read_template(**kwargs): + tplC = '' + with open(kwargs['pnl'], 'r') as tC: + tplC = tC.read() + return tplC + +def output(outHandle, **kwargs): + templateCode = read_template(pnl=kwargs['pnl']) + outHandle.write(templateCode) diff --git a/monitor/grafana/provisioning/_dashboard_generator/templates/footer b/monitor/grafana/provisioning/_dashboard_generator/templates7/footer similarity index 100% rename from monitor/grafana/provisioning/_dashboard_generator/templates/footer rename to monitor/grafana/provisioning/_dashboard_generator/templates7/footer diff --git a/monitor/grafana/provisioning/_dashboard_generator/templates/header b/monitor/grafana/provisioning/_dashboard_generator/templates7/header similarity index 100% rename from monitor/grafana/provisioning/_dashboard_generator/templates/header rename to monitor/grafana/provisioning/_dashboard_generator/templates7/header diff --git a/monitor/grafana/provisioning/_dashboard_generator/templates/panelBanner b/monitor/grafana/provisioning/_dashboard_generator/templates7/panelBanner similarity index 100% rename from monitor/grafana/provisioning/_dashboard_generator/templates/panelBanner rename to monitor/grafana/provisioning/_dashboard_generator/templates7/panelBanner diff --git a/monitor/grafana/provisioning/_dashboard_generator/templates/panelFooter b/monitor/grafana/provisioning/_dashboard_generator/templates7/panelFooter similarity index 100% rename from monitor/grafana/provisioning/_dashboard_generator/templates/panelFooter rename to monitor/grafana/provisioning/_dashboard_generator/templates7/panelFooter diff --git a/monitor/grafana/provisioning/_dashboard_generator/templates/panelHeader b/monitor/grafana/provisioning/_dashboard_generator/templates7/panelHeader similarity index 100% rename from monitor/grafana/provisioning/_dashboard_generator/templates/panelHeader rename to monitor/grafana/provisioning/_dashboard_generator/templates7/panelHeader diff --git a/monitor/grafana/provisioning/_dashboard_generator/templates/panelSingle b/monitor/grafana/provisioning/_dashboard_generator/templates7/panelSingle similarity index 100% rename from monitor/grafana/provisioning/_dashboard_generator/templates/panelSingle rename to monitor/grafana/provisioning/_dashboard_generator/templates7/panelSingle diff --git a/monitor/grafana/provisioning/_dashboard_generator/templates/panelSingleHadoopUsed b/monitor/grafana/provisioning/_dashboard_generator/templates7/panelSingleHadoopUsed similarity index 100% rename from monitor/grafana/provisioning/_dashboard_generator/templates/panelSingleHadoopUsed rename to monitor/grafana/provisioning/_dashboard_generator/templates7/panelSingleHadoopUsed diff --git a/monitor/grafana/provisioning/_dashboard_generator/templates7/panelSpacer b/monitor/grafana/provisioning/_dashboard_generator/templates7/panelSpacer new file mode 100644 index 0000000..46a5bff --- /dev/null +++ b/monitor/grafana/provisioning/_dashboard_generator/templates7/panelSpacer @@ -0,0 +1,23 @@ + { + "gridPos": { + "h": <h>, + "w": <w>, + "x": <x>, + "y": <y> + }, + "id": <id>, + "links": [], + "title": " ", + "type": "text", + "transparent": true, + "options": { + "mode": "markdown", + "content": "" + }, + "pluginVersion": "7.5.7", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "datasource": null + }, diff --git a/monitor/grafana/provisioning/_dashboard_generator/templates/panelStat b/monitor/grafana/provisioning/_dashboard_generator/templates7/panelStat similarity index 100% rename from monitor/grafana/provisioning/_dashboard_generator/templates/panelStat rename to monitor/grafana/provisioning/_dashboard_generator/templates7/panelStat diff --git a/monitor/grafana/provisioning/_dashboard_generator/templates/panelStatHadoopUsed b/monitor/grafana/provisioning/_dashboard_generator/templates7/panelStatHadoopUsed similarity index 100% rename from monitor/grafana/provisioning/_dashboard_generator/templates/panelStatHadoopUsed rename to monitor/grafana/provisioning/_dashboard_generator/templates7/panelStatHadoopUsed diff --git a/monitor/grafana/provisioning/_dashboard_generator/templates/panelTitle b/monitor/grafana/provisioning/_dashboard_generator/templates7/panelTitle similarity index 100% rename from monitor/grafana/provisioning/_dashboard_generator/templates/panelTitle rename to monitor/grafana/provisioning/_dashboard_generator/templates7/panelTitle diff --git a/monitor/grafana/provisioning/dashboards/wa_status.json-template b/monitor/grafana/provisioning/dashboards/wa_status.json-template index 7202772..8b724c7 100644 --- a/monitor/grafana/provisioning/dashboards/wa_status.json-template +++ b/monitor/grafana/provisioning/dashboards/wa_status.json-template @@ -18,36 +18,15 @@ "id": 5, "links": [], "panels": [ - { - "type": "text", - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "mode": "markdown", - "content": "# Storage" - }, - "transparent": true, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "datasource": null, - "pluginVersion": "7.5.7" - }, { "content": "", "gridPos": { "h": 1, "w": 8, "x": 0, - "y": 2 + "y": 0 }, - "id": 2, + "id": 1, "links": [], "mode": "markdown", "title": "Hadoop 0.20", @@ -92,9 +71,9 @@ "h": 2, "w": 2, "x": 0, - "y": 3 + "y": 1 }, - "id": 3, + "id": 2, "links": [], "maxDataPoints": 100, "options": { @@ -168,9 +147,9 @@ "h": 2, "w": 2, "x": 2, - "y": 3 + "y": 1 }, - "id": 4, + "id": 3, "links": [], "maxDataPoints": 100, "options": { @@ -244,9 +223,9 @@ "h": 2, "w": 2, "x": 4, - "y": 3 + "y": 1 }, - "id": 5, + "id": 4, "links": [], "maxDataPoints": 100, "options": { @@ -286,9 +265,9 @@ "h": 4, "w": 2, "x": 6, - "y": 3 + "y": 1 }, - "id": 6, + "id": 5, "links": [], "maxDataPoints": 100, "targets": [ @@ -331,7 +310,7 @@ "color": "#ED8027" }, { - "value": 90, + "value": 96, "color": "#D44A3A" } ] @@ -393,9 +372,9 @@ "h": 2, "w": 2, "x": 0, - "y": 5 + "y": 3 }, - "id": 7, + "id": 6, "links": [], "maxDataPoints": 100, "options": { @@ -469,9 +448,9 @@ "h": 2, "w": 2, "x": 2, - "y": 5 + "y": 3 }, - "id": 8, + "id": 7, "links": [], "maxDataPoints": 100, "options": { @@ -545,9 +524,9 @@ "h": 2, "w": 2, "x": 4, - "y": 5 + "y": 3 }, - "id": 9, + "id": 8, "links": [], "maxDataPoints": 100, "options": { @@ -588,9 +567,9 @@ "h": 1, "w": 8, "x": 8, - "y": 2 + "y": 0 }, - "id": 10, + "id": 9, "links": [], "mode": "markdown", "title": "Gluster", @@ -635,9 +614,9 @@ "h": 2, "w": 2, "x": 8, - "y": 3 + "y": 1 }, - "id": 11, + "id": 10, "links": [], "maxDataPoints": 100, "options": { @@ -711,9 +690,9 @@ "h": 2, "w": 2, "x": 10, - "y": 3 + "y": 1 }, - "id": 12, + "id": 11, "links": [], "maxDataPoints": 100, "options": { @@ -787,9 +766,9 @@ "h": 2, "w": 2, "x": 12, - "y": 3 + "y": 1 }, - "id": 13, + "id": 12, "links": [], "maxDataPoints": 100, "options": { @@ -863,9 +842,9 @@ "h": 2, "w": 2, "x": 14, - "y": 3 + "y": 1 }, - "id": 14, + "id": 13, "links": [], "maxDataPoints": 100, "options": { @@ -900,13 +879,36 @@ "interval": null, "datasource": null }, + { + "gridPos": { + "h": 2, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 14, + "links": [], + "title": " ", + "type": "text", + "transparent": true, + "options": { + "mode": "markdown", + "content": "" + }, + "pluginVersion": "7.5.7", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "datasource": null + }, { "content": "", "gridPos": { "h": 1, "w": 8, "x": 16, - "y": 2 + "y": 0 }, "id": 15, "links": [], @@ -953,7 +955,7 @@ "h": 2, "w": 2, "x": 16, - "y": 3 + "y": 1 }, "id": 16, "links": [], @@ -1029,7 +1031,7 @@ "h": 2, "w": 2, "x": 18, - "y": 3 + "y": 1 }, "id": 17, "links": [], @@ -1105,7 +1107,7 @@ "h": 2, "w": 2, "x": 20, - "y": 3 + "y": 1 }, "id": 18, "links": [], @@ -1147,7 +1149,7 @@ "h": 4, "w": 2, "x": 22, - "y": 3 + "y": 1 }, "id": 19, "links": [], @@ -1192,7 +1194,7 @@ "color": "#ED8027" }, { - "value": 90, + "value": 96, "color": "#D44A3A" } ] @@ -1254,7 +1256,7 @@ "h": 2, "w": 2, "x": 16, - "y": 5 + "y": 3 }, "id": 20, "links": [], @@ -1330,7 +1332,7 @@ "h": 2, "w": 2, "x": 18, - "y": 5 + "y": 3 }, "id": 21, "links": [], @@ -1406,7 +1408,7 @@ "h": 2, "w": 2, "x": 20, - "y": 5 + "y": 3 }, "id": 22, "links": [], @@ -1420,7 +1422,7 @@ "fields": "" }, "orientation": "horizontal", - "textMode": "value", + "textMode": "none", "colorMode": "background", "graphMode": "none", "justifyMode": "auto" @@ -1443,36 +1445,15 @@ "interval": null, "datasource": null }, - { - "type": "text", - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 23, - "options": { - "mode": "markdown", - "content": "# WA Systems" - }, - "transparent": true, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "datasource": null, - "pluginVersion": "7.5.7" - }, { "content": "", "gridPos": { "h": 1, "w": 8, "x": 0, - "y": 8 + "y": 5 }, - "id": 24, + "id": 23, "links": [], "mode": "markdown", "title": "Infrastructure", @@ -1517,9 +1498,9 @@ "h": 2, "w": 2, "x": 0, - "y": 9 + "y": 6 }, - "id": 25, + "id": 24, "links": [], "maxDataPoints": 100, "options": { @@ -1593,9 +1574,9 @@ "h": 2, "w": 2, "x": 2, - "y": 9 + "y": 6 }, - "id": 26, + "id": 25, "links": [], "maxDataPoints": 100, "options": { @@ -1669,9 +1650,9 @@ "h": 2, "w": 2, "x": 4, - "y": 9 + "y": 6 }, - "id": 27, + "id": 26, "links": [], "maxDataPoints": 100, "options": { @@ -1745,9 +1726,9 @@ "h": 2, "w": 2, "x": 6, - "y": 9 + "y": 6 }, - "id": 28, + "id": 27, "links": [], "maxDataPoints": 100, "options": { @@ -1821,9 +1802,9 @@ "h": 2, "w": 2, "x": 0, - "y": 11 + "y": 8 }, - "id": 29, + "id": 28, "links": [], "maxDataPoints": 100, "options": { @@ -1843,7 +1824,7 @@ "pluginVersion": "7.1.3", "targets": [ { - "expr": "sum(last_over_time(recent_connections{job=\"ldl_rr\"}[30m]))", + "expr": "sum(last_over_time(recent_connections{job=\"ldl_rr\",instance=~\"DLS-.+\"}[30m]))", "legendFormat": "", "interval": "", "format": "time_series", @@ -1864,9 +1845,9 @@ "h": 1, "w": 8, "x": 8, - "y": 8 + "y": 5 }, - "id": 30, + "id": 29, "links": [], "mode": "markdown", "title": "Services", @@ -1911,9 +1892,9 @@ "h": 2, "w": 2, "x": 8, - "y": 9 + "y": 6 }, - "id": 31, + "id": 30, "links": [], "maxDataPoints": 100, "options": { @@ -1987,9 +1968,9 @@ "h": 2, "w": 2, "x": 10, - "y": 9 + "y": 6 }, - "id": 32, + "id": 31, "links": [], "maxDataPoints": 100, "options": { @@ -2063,9 +2044,9 @@ "h": 2, "w": 2, "x": 12, - "y": 9 + "y": 6 }, - "id": 33, + "id": 32, "links": [], "maxDataPoints": 100, "options": { @@ -2139,9 +2120,9 @@ "h": 2, "w": 2, "x": 14, - "y": 9 + "y": 6 }, - "id": 34, + "id": 33, "links": [], "maxDataPoints": 100, "options": { @@ -2176,13 +2157,36 @@ "interval": null, "datasource": null }, + { + "gridPos": { + "h": 2, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 34, + "links": [], + "title": " ", + "type": "text", + "transparent": true, + "options": { + "mode": "markdown", + "content": "" + }, + "pluginVersion": "7.5.7", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "datasource": null + }, { "content": "", "gridPos": { "h": 1, "w": 8, "x": 16, - "y": 8 + "y": 5 }, "id": 35, "links": [], @@ -2229,7 +2233,7 @@ "h": 2, "w": 2, "x": 16, - "y": 9 + "y": 6 }, "id": 36, "links": [], @@ -2305,7 +2309,7 @@ "h": 2, "w": 2, "x": 18, - "y": 9 + "y": 6 }, "id": 37, "links": [], @@ -2381,7 +2385,7 @@ "h": 2, "w": 2, "x": 20, - "y": 9 + "y": 6 }, "id": 38, "links": [], @@ -2457,7 +2461,7 @@ "h": 2, "w": 2, "x": 22, - "y": 9 + "y": 6 }, "id": 39, "links": [], @@ -2533,7 +2537,7 @@ "h": 2, "w": 2, "x": 16, - "y": 11 + "y": 8 }, "id": 40, "links": [], @@ -2570,36 +2574,15 @@ "interval": null, "datasource": null }, - { - "type": "text", - "gridPos": { - "h": 2, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 41, - "options": { - "mode": "markdown", - "content": "# WA Services" - }, - "transparent": true, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "datasource": null, - "pluginVersion": "7.5.7" - }, { "content": "", "gridPos": { "h": 1, "w": 8, "x": 0, - "y": 14 + "y": 10 }, - "id": 42, + "id": 41, "links": [], "mode": "markdown", "title": "Ingest & Metadata", @@ -2644,9 +2627,9 @@ "h": 2, "w": 2, "x": 0, - "y": 15 + "y": 11 }, - "id": 43, + "id": 42, "links": [], "maxDataPoints": 100, "options": { @@ -2695,10 +2678,10 @@ }, { "value": 0.1, - "color": "#ba43a9" + "color": "#ED8027" }, { - "value": 1.1, + "value": 1, "color": "#D44A3A" } ] @@ -2720,9 +2703,9 @@ "h": 2, "w": 2, "x": 2, - "y": 15 + "y": 11 }, - "id": 44, + "id": 43, "links": [], "maxDataPoints": 100, "options": { @@ -2796,9 +2779,9 @@ "h": 2, "w": 2, "x": 4, - "y": 15 + "y": 11 }, - "id": 45, + "id": 44, "links": [], "maxDataPoints": 100, "options": { @@ -2872,9 +2855,9 @@ "h": 2, "w": 2, "x": 6, - "y": 15 + "y": 11 }, - "id": 46, + "id": 45, "links": [], "maxDataPoints": 100, "options": { @@ -2948,9 +2931,9 @@ "h": 2, "w": 2, "x": 0, - "y": 17 + "y": 13 }, - "id": 47, + "id": 46, "links": [], "maxDataPoints": 100, "options": { @@ -2991,9 +2974,9 @@ "h": 1, "w": 8, "x": 8, - "y": 14 + "y": 10 }, - "id": 48, + "id": 47, "links": [], "mode": "markdown", "title": "TrackDB", @@ -3038,9 +3021,9 @@ "h": 2, "w": 4, "x": 8, - "y": 15 + "y": 10 }, - "id": 49, + "id": 48, "links": [], "maxDataPoints": 100, "options": { @@ -3070,7 +3053,7 @@ } ], "type": "stat", - "title": "trackdb refresh", + "title": "Refresh", "cacheTimeout": null, "interval": null, "datasource": null @@ -3114,9 +3097,9 @@ "h": 2, "w": 4, "x": 12, - "y": 15 + "y": 10 }, - "id": 50, + "id": 49, "links": [], "maxDataPoints": 100, "options": { @@ -3146,7 +3129,7 @@ } ], "type": "stat", - "title": "trackdb numFound", + "title": "numFound", "cacheTimeout": null, "interval": null, "datasource": null @@ -3156,10 +3139,10 @@ "gridPos": { "h": 1, "w": 8, - "x": 16, - "y": 14 + "x": 16.0, + "y": 10 }, - "id": 51, + "id": 50, "links": [], "mode": "markdown", "title": "Discovery & Access", @@ -3203,10 +3186,10 @@ "gridPos": { "h": 2, "w": 2, - "x": 16, - "y": 15 + "x": 16.0, + "y": 11 }, - "id": 52, + "id": 51, "links": [], "maxDataPoints": 100, "options": { @@ -3279,10 +3262,10 @@ "gridPos": { "h": 2, "w": 2, - "x": 18, - "y": 15 + "x": 18.0, + "y": 11 }, - "id": 53, + "id": 52, "links": [], "maxDataPoints": 100, "options": { @@ -3355,10 +3338,10 @@ "gridPos": { "h": 2, "w": 2, - "x": 20, - "y": 15 + "x": 20.0, + "y": 11 }, - "id": 54, + "id": 53, "links": [], "maxDataPoints": 100, "options": { @@ -3407,10 +3390,10 @@ }, { "value": 0.1, - "color": "#ba43a9" + "color": "#ED8027" }, { - "value": 1.1, + "value": 1, "color": "#D44A3A" } ] @@ -3431,10 +3414,10 @@ "gridPos": { "h": 2, "w": 2, - "x": 22, - "y": 15 + "x": 22.0, + "y": 11 }, - "id": 55, + "id": 54, "links": [], "maxDataPoints": 100, "options": { @@ -3507,10 +3490,10 @@ "gridPos": { "h": 2, "w": 2, - "x": 16, - "y": 17 + "x": 16.0, + "y": 13 }, - "id": 56, + "id": 55, "links": [], "maxDataPoints": 100, "options": { @@ -3583,10 +3566,10 @@ "gridPos": { "h": 2, "w": 2, - "x": 18, - "y": 17 + "x": 18.0, + "y": 13 }, - "id": 57, + "id": 56, "links": [], "maxDataPoints": 100, "options": {