From f421656e2e1be42afd41d92568ca15fe9c13c648 Mon Sep 17 00:00:00 2001
From: Jan Dobes <jdobes@redhat.com>
Date: Thu, 15 Aug 2024 16:35:13 +0200
Subject: [PATCH] chore: split some evaluator graphs to upload and recalc

---
 ...nsights-vulnerability-engine.configmap.yml | 461 ++++++++++++++----
 1 file changed, 357 insertions(+), 104 deletions(-)

diff --git a/monitoring/grafana/dashboards/grafana-dashboard-clouddot-insights-vulnerability-engine.configmap.yml b/monitoring/grafana/dashboards/grafana-dashboard-clouddot-insights-vulnerability-engine.configmap.yml
index b8f2f7d5a..393cfec5b 100644
--- a/monitoring/grafana/dashboards/grafana-dashboard-clouddot-insights-vulnerability-engine.configmap.yml
+++ b/monitoring/grafana/dashboards/grafana-dashboard-clouddot-insights-vulnerability-engine.configmap.yml
@@ -4777,7 +4777,7 @@ data:
               "refId": "D"
             }
           ],
-          "title": "evaluator processing rate",
+          "title": "upload evaluator processing rate",
           "type": "timeseries"
         },
         {
@@ -5271,7 +5271,300 @@ data:
                 "type": "prometheus",
                 "uid": "pKSsqZB7k"
               },
-              "expr": "sum(increase(ve_evaluator_vmaas_evaluation_seconds_bucket[5m])) by (le)",
+              "editorMode": "code",
+              "expr": "sum(increase(ve_evaluator_vmaas_evaluation_seconds_bucket{job=\"vulnerability-engine-evaluator-upload-service\"}[5m])) by (le)",
+              "format": "heatmap",
+              "groupBy": [
+                {
+                  "params": [
+                    "$__interval"
+                  ],
+                  "type": "time"
+                },
+                {
+                  "params": [
+                    "null"
+                  ],
+                  "type": "fill"
+                }
+              ],
+              "interval": "",
+              "intervalFactor": 10,
+              "legendFormat": "{{le}}",
+              "orderByTime": "ASC",
+              "policy": "default",
+              "range": true,
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "mean"
+                  }
+                ]
+              ],
+              "tags": []
+            }
+          ],
+          "title": "upload evaluator processing duration",
+          "type": "heatmap"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$datasource"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "log": 10,
+                  "type": "log"
+                },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "links": [],
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 97
+          },
+          "id": 162,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "multi",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "10.4.1",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$datasource"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(ve_evaluator_message_parse_error_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
+              "hide": false,
+              "legendFormat": "message parse error",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$datasource"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(ve_evaluator_inventory_not_found_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
+              "hide": false,
+              "legendFormat": "inventory not found",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$datasource"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(ve_evaluator_unknown_msg_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
+              "hide": false,
+              "legendFormat": "unknown kafka message",
+              "range": true,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$datasource"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(ve_evaluator_vmaas_errors_skip_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
+              "hide": false,
+              "legendFormat": "vmaas error skipped",
+              "range": true,
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$datasource"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(ve_evaluator_data_skip_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
+              "hide": false,
+              "legendFormat": "skipped due to empty vmaas and advisor results",
+              "range": true,
+              "refId": "E"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$datasource"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(ve_evaluator_evaluated_earlier_skip_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "evaluated earlier",
+              "range": true,
+              "refId": "F"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "$datasource"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(ve_evaluator_recalc_packages_unchanged_skip_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "recalc packages not changed",
+              "range": true,
+              "refId": "G"
+            }
+          ],
+          "title": "re-calc evaluator skipped messages rate",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "$datasource"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "custom": {
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "scaleDistribution": {
+                  "type": "linear"
+                }
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 104
+          },
+          "id": 190,
+          "options": {
+            "calculate": false,
+            "calculation": {},
+            "cellGap": 2,
+            "cellValues": {},
+            "color": {
+              "exponent": 0.5,
+              "fill": "dark-orange",
+              "mode": "scheme",
+              "reverse": false,
+              "scale": "exponential",
+              "scheme": "Magma",
+              "steps": 128
+            },
+            "exemplars": {
+              "color": "rgba(255,0,255,0.7)"
+            },
+            "filterValues": {
+              "le": 1e-9
+            },
+            "legend": {
+              "show": false,
+              "showLegend": true
+            },
+            "rowsFrame": {
+              "layout": "auto"
+            },
+            "showValue": "never",
+            "tooltip": {
+              "mode": "single",
+              "showColorScale": false,
+              "yHistogram": false
+            },
+            "yAxis": {
+              "axisPlacement": "left",
+              "min": "0",
+              "reverse": false,
+              "unit": "s"
+            }
+          },
+          "pluginVersion": "10.4.1",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "pKSsqZB7k"
+              },
+              "editorMode": "code",
+              "expr": "sum(increase(ve_evaluator_vmaas_evaluation_seconds_bucket{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m])) by (le)",
               "format": "heatmap",
               "groupBy": [
                 {
@@ -5292,6 +5585,7 @@ data:
               "legendFormat": "{{le}}",
               "orderByTime": "ASC",
               "policy": "default",
+              "range": true,
               "refId": "A",
               "resultFormat": "time_series",
               "select": [
@@ -5311,7 +5605,7 @@ data:
               "tags": []
             }
           ],
-          "title": "evaluator processing duration",
+          "title": "re-calc evaluator processing duration",
           "type": "heatmap"
         },
         {
@@ -5378,7 +5672,7 @@ data:
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 97
+            "y": 104
           },
           "id": 188,
           "options": {
@@ -5402,7 +5696,7 @@ data:
               },
               "editorMode": "code",
               "exemplar": true,
-              "expr": "histogram_quantile(0.5, sum(rate(ve_evaluator_evaluation_part_seconds_bucket{job=~\"vulnerability.*-evaluator-.*\"}[5m])) by (part, le))",
+              "expr": "histogram_quantile(0.5, sum(rate(ve_evaluator_evaluation_part_seconds_bucket{job=~\"vulnerability.*-evaluator-upload-.*\"}[5m])) by (part, le))",
               "format": "time_series",
               "groupBy": [
                 {
@@ -5443,7 +5737,7 @@ data:
               "tags": []
             }
           ],
-          "title": "evaluator part duration median",
+          "title": "upload evaluator part duration median",
           "type": "timeseries"
         },
         {
@@ -5510,7 +5804,7 @@ data:
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 104
+            "y": 111
           },
           "id": 65,
           "options": {
@@ -5585,8 +5879,8 @@ data:
                 "axisBorderShow": false,
                 "axisCenteredZero": false,
                 "axisColorMode": "text",
-                "axisLabel": "",
-                "axisPlacement": "auto",
+                "axisLabel": "duration",
+                "axisPlacement": "left",
                 "barAlignment": 0,
                 "drawStyle": "line",
                 "fillOpacity": 10,
@@ -5601,8 +5895,7 @@ data:
                 "lineWidth": 1,
                 "pointSize": 5,
                 "scaleDistribution": {
-                  "log": 10,
-                  "type": "log"
+                  "type": "linear"
                 },
                 "showPoints": "never",
                 "spanNulls": false,
@@ -5614,7 +5907,6 @@ data:
                   "mode": "off"
                 }
               },
-              "links": [],
               "mappings": [],
               "thresholds": {
                 "mode": "absolute",
@@ -5629,7 +5921,7 @@ data:
                   }
                 ]
               },
-              "unit": "short"
+              "unit": "s"
             },
             "overrides": []
           },
@@ -5637,9 +5929,9 @@ data:
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 104
+            "y": 111
           },
-          "id": 162,
+          "id": 189,
           "options": {
             "legend": {
               "calcs": [],
@@ -5649,7 +5941,7 @@ data:
             },
             "tooltip": {
               "mode": "multi",
-              "sort": "none"
+              "sort": "desc"
             }
           },
           "pluginVersion": "10.4.1",
@@ -5657,91 +5949,52 @@ data:
             {
               "datasource": {
                 "type": "prometheus",
-                "uid": "$datasource"
-              },
-              "editorMode": "code",
-              "expr": "sum(rate(ve_evaluator_message_parse_error_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
-              "hide": false,
-              "legendFormat": "message parse error",
-              "range": true,
-              "refId": "A"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "$datasource"
-              },
-              "editorMode": "code",
-              "expr": "sum(rate(ve_evaluator_inventory_not_found_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
-              "hide": false,
-              "legendFormat": "inventory not found",
-              "range": true,
-              "refId": "B"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "$datasource"
-              },
-              "editorMode": "code",
-              "expr": "sum(rate(ve_evaluator_unknown_msg_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
-              "hide": false,
-              "legendFormat": "unknown kafka message",
-              "range": true,
-              "refId": "C"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "$datasource"
-              },
-              "editorMode": "code",
-              "expr": "sum(rate(ve_evaluator_vmaas_errors_skip_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
-              "hide": false,
-              "legendFormat": "vmaas error skipped",
-              "range": true,
-              "refId": "D"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "$datasource"
-              },
-              "editorMode": "code",
-              "expr": "sum(rate(ve_evaluator_data_skip_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
-              "hide": false,
-              "legendFormat": "skipped due to empty vmaas and advisor results",
-              "range": true,
-              "refId": "E"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "$datasource"
-              },
-              "editorMode": "code",
-              "expr": "sum(rate(ve_evaluator_evaluated_earlier_skip_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
-              "hide": false,
-              "instant": false,
-              "legendFormat": "evaluated earlier",
-              "range": true,
-              "refId": "F"
-            },
-            {
-              "datasource": {
-                "type": "prometheus",
-                "uid": "$datasource"
+                "uid": "pKSsqZB7k"
               },
               "editorMode": "code",
-              "expr": "sum(rate(ve_evaluator_recalc_packages_unchanged_skip_total{job=\"vulnerability-engine-evaluator-recalc-service\"}[5m]))",
-              "hide": false,
-              "instant": false,
-              "legendFormat": "recalc packages not changed",
+              "exemplar": true,
+              "expr": "histogram_quantile(0.5, sum(rate(ve_evaluator_evaluation_part_seconds_bucket{job=~\"vulnerability.*-evaluator-recalc-.*\"}[5m])) by (part, le))",
+              "format": "time_series",
+              "groupBy": [
+                {
+                  "params": [
+                    "$__interval"
+                  ],
+                  "type": "time"
+                },
+                {
+                  "params": [
+                    "null"
+                  ],
+                  "type": "fill"
+                }
+              ],
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{part}}",
+              "orderByTime": "ASC",
+              "policy": "default",
               "range": true,
-              "refId": "G"
+              "refId": "A",
+              "resultFormat": "time_series",
+              "select": [
+                [
+                  {
+                    "params": [
+                      "value"
+                    ],
+                    "type": "field"
+                  },
+                  {
+                    "params": [],
+                    "type": "mean"
+                  }
+                ]
+              ],
+              "tags": []
             }
           ],
-          "title": "re-calc evaluator skipped messages rate",
+          "title": "re-calc evaluator part duration median",
           "type": "timeseries"
         },
         {
@@ -5772,7 +6025,7 @@ data:
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 111
+            "y": 118
           },
           "id": 141,
           "options": {
@@ -5904,7 +6157,7 @@ data:
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 111
+            "y": 118
           },
           "id": 104,
           "options": {
@@ -6007,7 +6260,7 @@ data:
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 118
+            "y": 125
           },
           "id": 59,
           "options": {
@@ -6144,7 +6397,7 @@ data:
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 118
+            "y": 125
           },
           "id": 140,
           "options": {
@@ -6272,7 +6525,7 @@ data:
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 125
+            "y": 132
           },
           "id": 164,
           "options": {
@@ -6363,7 +6616,7 @@ data:
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 125
+            "y": 132
           },
           "id": 159,
           "options": {
@@ -6480,7 +6733,7 @@ data:
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 132
+            "y": 139
           },
           "id": 158,
           "options": {
@@ -6670,7 +6923,7 @@ data:
       "timezone": "utc",
       "title": "vulnerability-engine",
       "uid": "vUgt4Eumk",
-      "version": 4,
+      "version": 5,
       "weekStart": ""
     }
 kind: ConfigMap