diff --git a/README.md b/README.md index b13f91721..ed6f7f360 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,16 @@ TODO: Add the rest of the standard endpoints add unique endpoints (e.g. getSlice `make test` will run the unit tests `make test` setups a clean `vulcanize_testing` db +## Monitoring + +* Enable http server and metrics using parameters `--http --metrics` +* ipld-eth-server exposes prometheus metrics at `/metric` endpoint +* start prometheus using `monitoring/prometheus.yml` config (`prometheus --config.file=monitoring/prometheus.yml`) +* start grafana, connect to prometheus datasource and import dashboard from `monitoring/grafana/dashboard_main.json` + +![](monitoring/grafana.png) + + ## Contributing Contributions are welcome! diff --git a/cmd/root.go b/cmd/root.go index 2e8c86b7c..00fba60e1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -111,7 +111,7 @@ func init() { rootCmd.PersistentFlags().Bool("http", false, "enable http service for prometheus") rootCmd.PersistentFlags().String("http-addr", "127.0.0.1", "http host for prometheus") - rootCmd.PersistentFlags().String("http-port", "8080", "http port for prometheus") + rootCmd.PersistentFlags().String("http-port", "8090", "http port for prometheus") viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile")) viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) diff --git a/monitoring/grafana.png b/monitoring/grafana.png new file mode 100644 index 000000000..3d1220e12 Binary files /dev/null and b/monitoring/grafana.png differ diff --git a/monitoring/grafana/dashboard_main.json b/monitoring/grafana/dashboard_main.json new file mode 100644 index 000000000..83ed1a0b3 --- /dev/null +++ b/monitoring/grafana/dashboard_main.json @@ -0,0 +1,353 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.2.1", + "targets": [ + { + "expr": "ipld_eth_server_ws_count", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Websocket Connection Count", + "type": "gauge" + }, + { + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.2.1", + "targets": [ + { + "expr": "ipld_eth_server_ipc_count", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "IPC Connection Count", + "type": "gauge" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(ipld_eth_server_http_count[1m])", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP requests per second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {}, + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.9, rate(ipld_eth_server_http_duration_bucket[1m]))", + "hide": false, + "interval": "", + "legendFormat": "0.9", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.75, rate(ipld_eth_server_http_duration_bucket[1m]))", + "interval": "", + "legendFormat": "0.75", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.5, rate(ipld_eth_server_http_duration_bucket[1m]))", + "interval": "", + "legendFormat": "0.5", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP Requests duration (percentile)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "ipld-eth-server", + "uid": "lFVEvNtGk", + "version": 7 +} diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml new file mode 100644 index 000000000..79b4d6203 --- /dev/null +++ b/monitoring/prometheus.yml @@ -0,0 +1,7 @@ +global: + scrape_interval: 10s + +scrape_configs: + - job_name: 'ipld-eth-server' + static_configs: + - targets: ['localhost:8090'] diff --git a/pkg/prom/prom.go b/pkg/prom/prom.go index f17ef5464..72a282ce0 100644 --- a/pkg/prom/prom.go +++ b/pkg/prom/prom.go @@ -44,7 +44,7 @@ func Init() { Namespace: namespace, Subsystem: subsystemWS, Name: "count", - Help: "websocket conntection count", + Help: "websocket connection count", }) ipcCount = promauto.NewGauge(prometheus.GaugeOpts{