Add throughput and SLO metrics in the tags and tag values endpoints (#…

…4148) propagate the bytes read from the storage layer to the frontend, and use bytes read to compute throughout, and use that in SLO computation for metadata endpoints. metadata SLO thresholds can be configured via `metadata_slo` config. ``` search: metadata_slo: duration_slo: 5s throughput_bytes_slo: 1.073741824e+09 ``` we will also return the `metrics` in the response of all the metadata endpoints: - `/search/tags` - `/v2/search/tags` - `/search/tag/<tagName>/values` - `/v2/search/tag/<tagName>/values` here is what the it looks like in the response: ```json { "<existing keys>": "<existing response>", "metrics": { "inspectedBytes": "630188" } } ``` we return `"metrics": {}` when response is empty or only contains `intrinsics` it will also expose these new label to existing metrics with `op="metadata"` label - total metadata queries counter - metadata queries within SLO counter - metadata queries throughput histogram
grafana · Oct 15, 2024 · 327c964 · 327c964
1 parent e37f481
commit 327c964
Show file tree

Hide file tree

Showing 54 changed files with 2,171 additions and 701 deletions.
diff --git a/.gitignore b/.gitignore
@@ -20,5 +20,6 @@
 /tempodb/encoding/benchmark_block
 private-key.key
 integration/e2e/e2e_integration_test[0-9]*
+integration/e2e/deployments/e2e_integration_test[0-9]*
 .tempo.yaml
 /tmp
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@
 * [ENHANCEMENT] Changed log level from INFO to DEBUG for the TempoDB Find operation using traceId to reduce excessive/unwanted logs in log search. [#4179](https://github.com/grafana/tempo/pull/4179) (@Aki0x137)
 * [ENHANCEMENT] tempo-query: separate tls settings for server and client [#4177](https://github.com/grafana/tempo/pull/4177) (@frzifus)
 * [ENHANCEMENT] Pushdown collection of results from generators in the querier [#4119](https://github.com/grafana/tempo/pull/4119) (@electron0zero)
+* [CHANGE] Add throughput and SLO metrics in the tags and tag values endpoints [#4148](https://github.com/grafana/tempo/pull/4148) (@electron0zero)
 * [ENHANCEMENT] Send semver version in api/stattus/buildinfo for cloud deployments [#4110](https://github.com/grafana/tempo/pull/4110) [@Aki0x137]
 * [ENHANCEMENT] Speedup tempo-query trace search by allowing parallel queries [#4159](https://github.com/grafana/tempo/pull/4159) (@pavolloffay)
 * [ENHANCEMENT] Speedup DistinctString and ScopedDistinctString collectors [#4109](https://github.com/grafana/tempo/pull/4109) (@electron0zero)

diff --git a/docs/sources/tempo/api_docs/_index.md b/docs/sources/tempo/api_docs/_index.md
@@ -343,6 +343,9 @@ $ curl -G -s http://localhost:3200/api/search/tags?scope=span  | jq
     "starter",
     "version"
   ]
+  "metrics": {
+    "inspectedBytes": "630188"
+  }
 }
 ```
 
@@ -391,11 +394,9 @@ $ curl -G -s http://localhost:3200/api/v2/search/tags  | jq
 {
   "scopes": [
     {
-      "name": "span",
+      "name": "link",
       "tags": [
-        "article.count",
-        "http.flavor",
-        "http.method",
+        "link-type"
       ]
     },
     {
@@ -405,16 +406,70 @@ $ curl -G -s http://localhost:3200/api/v2/search/tags  | jq
         "service.name"
       ]
     },
+    {
+      "name": "span",
+      "tags": [
+        "article.count",
+        "http.flavor",
+        "http.method",
+        "http.request.header.accept",
+        "http.request_content_length",
+        "http.response.header.content-type",
+        "http.response_content_length",
+        "http.scheme",
+        "http.status_code",
+        "http.target",
+        "http.url",
+        "net.host.name",
+        "net.host.port",
+        "net.peer.name",
+        "net.peer.port",
+        "net.sock.family",
+        "net.sock.host.addr",
+        "net.sock.peer.addr",
+        "net.transport",
+        "numbers",
+        "one"
+      ]
+    },
     {
       "name": "intrinsic",
       "tags": [
         "duration",
+        "event:name",
+        "event:timeSinceStart",
+        "instrumentation:name",
+        "instrumentation:version",
         "kind",
         "name",
-        "status"
+        "rootName",
+        "rootServiceName",
+        "span:duration",
+        "span:kind",
+        "span:name",
+        "span:status",
+        "span:statusMessage",
+        "status",
+        "statusMessage",
+        "trace:duration",
+        "trace:rootName",
+        "trace:rootService",
+        "traceDuration"
+      ]
+    },
+    {
+      "name": "event",
+      "tags": [
+        "exception.escape",
+        "exception.message",
+        "exception.stacktrace",
+        "exception.type",
       ]
     }
-  ]
+  ],
+  "metrics": {
+    "inspectedBytes": "377046"
+  }
 }
 ```
 
@@ -440,13 +495,16 @@ This query returns all discovered values for the tag `service.name`.
 $ curl -G -s http://localhost:3200/api/search/tag/service.name/values  | jq
 {
   "tagValues": [
-    "adservice",
-    "cartservice",
-    "checkoutservice",
-    "frontend",
-    "productcatalogservice",
-    "recommendationservice"
-  ]
+    "article-service",
+    "auth-service",
+    "billing-service",
+    "cart-service",
+    "postgres",
+    "shop-backend"
+  ],
+  "metrics": {
+    "inspectedBytes": "431380"
+  }
 }
 ```
 
@@ -468,30 +526,37 @@ See [TraceQL]({{< relref "../traceql" >}}) documentation for more information.
 This example queries Tempo using curl and returns all discovered values for the tag `service.name`.
 
 ```bash
-$ curl http://localhost:3200/api/v2/search/tag/.service.name/values | jq .
+$ curl -G -s http://localhost:3200/api/v2/search/tag/.service.name/values | jq
 {
   "tagValues": [
     {
       "type": "string",
-      "value": "customer"
+      "value": "article-service"
+    },
+    {
+      "type": "string",
+      "value": "postgres"
     },
     {
       "type": "string",
-      "value": "mysql"
+      "value": "cart-service"
     },
     {
       "type": "string",
-      "value": "driver"
+      "value": "billing-service"
     },
     {
       "type": "string",
-      "value": "frontend"
+      "value": "shop-backend"
     },
     {
       "type": "string",
-      "value": "redis"
+      "value": "auth-service"
     }
-  ]
+  ],
+  "metrics": {
+    "inspectedBytes": "502756"
+  }
 }
 ```
 This endpoint can also receive `start` and `end` optional parameters. These parameters define the time range from which the tags are fetched

diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md
@@ -614,7 +614,7 @@ query_frontend:
 
         # If set to a non-zero value, it's value will be used to decide if query is within SLO or not.
         # Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data.
-        # NOTE: `duration_slo` and `throughput_bytes_slo` both must be configured for it to work
+        # NOTE: Requires `duration_slo` AND `throughput_bytes_slo` to be configured.
         [duration_slo: <duration> | default = 0s ]
 
         # If set to a non-zero value, it's value will be used to decide if query is within SLO or not.
@@ -623,6 +623,17 @@ query_frontend:
 
         # The number of shards to break ingester queries into.
         [ingester_shards]: <int> | default = 1]
+
+        # SLO configuration for Metadata (tags and tag values) endpoints.
+        metadata_slo:
+            # If set to a non-zero value, it's value will be used to decide if metadata query is within SLO or not.
+            # Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data.
+            # NOTE: Requires `duration_slo` AND `throughput_bytes_slo` to be configured.
+            [duration_slo: <duration> | default = 0s ]
+
+            # If set to a non-zero value, it's value will be used to decide if metadata query is within SLO or not.
+            # Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data.
+            [throughput_bytes_slo: <float> | default = 0 ]
 
     # Trace by ID lookup configuration
     trace_by_id:

diff --git a/docs/sources/tempo/configuration/manifest.md b/docs/sources/tempo/configuration/manifest.md
@@ -18,7 +18,7 @@ go run ./cmd/tempo --storage.trace.backend=local --storage.trace.local.path=/var
 ## Complete configuration
 
 {{< admonition type="note" >}}
-This manifest was generated on 2023-11-13.
+This manifest was generated on 2024-10-11.
 {{% /admonition %}}
 
 ```yaml
@@ -33,6 +33,7 @@ server:
     grpc_listen_address: ""
     grpc_listen_port: 9095
     grpc_listen_conn_limit: 0
+    proxy_protocol_enabled: false
     tls_cipher_suites: ""
     tls_min_version: ""
     http_tls_config:
@@ -70,6 +71,8 @@ server:
     grpc_server_min_time_between_pings: 10s
     grpc_server_ping_without_stream_allowed: true
     grpc_server_num_workers: 0
+    grpc_server_stats_tracking_enabled: true
+    grpc_server_recv_buffer_pools_enabled: false
     log_format: logfmt
     log_level: info
     log_source_ips_enabled: false
@@ -89,6 +92,7 @@ internal_server:
     grpc_listen_address: ""
     grpc_listen_port: 0
     grpc_listen_conn_limit: 0
+    proxy_protocol_enabled: false
     tls_cipher_suites: ""
     tls_min_version: ""
     http_tls_config:
@@ -126,6 +130,8 @@ internal_server:
     grpc_server_min_time_between_pings: 0s
     grpc_server_ping_without_stream_allowed: false
     grpc_server_num_workers: 0
+    grpc_server_stats_tracking_enabled: false
+    grpc_server_recv_buffer_pools_enabled: false
     log_format: logfmt
     log_level: info
     log_source_ips_enabled: false
@@ -314,7 +320,9 @@ query_frontend:
         max_duration: 3h0m0s
         query_backend_after: 30m0s
         interval: 5m0s
+        max_exemplars: 100
     multi_tenant_queries_enabled: true
+    response_consumers: 10
 compactor:
     ring:
         kvstore:
@@ -582,7 +590,7 @@ metrics_generator:
         path: ""
         v2_encoding: none
         search_encoding: none
-        ingestion_time_range_slack: 0s
+        ingestion_time_range_slack: 2m0s
         version: vParquet4
     metrics_ingestion_time_range_slack: 30s
     query_timeout: 30s
@@ -620,11 +628,13 @@ storage:
                 offset_index: false
         blocklist_poll: 5m0s
         blocklist_poll_concurrency: 50
+        blocklist_poll_tenant_concurrency: 0
         blocklist_poll_fallback: true
         blocklist_poll_tenant_index_builders: 2
         blocklist_poll_stale_tenant_index: 0s
         blocklist_poll_jitter_ms: 0
         blocklist_poll_tolerate_consecutive_errors: 1
+        blocklist_poll_tolerate_tenant_failures: 1
         empty_tenant_deletion_enabled: false
         empty_tenant_deletion_age: 0s
         backend: local
@@ -699,6 +709,9 @@ overrides:
             max_traces_per_user: 10000
         read:
             max_bytes_per_tag_values_query: 5000000
+        metrics_generator:
+            generate_native_histograms: classic
+            ingestion_time_range_slack: 0s
         global:
             max_bytes_per_trace: 5000000
     per_tenant_override_config: ""
@@ -788,6 +801,7 @@ memberlist:
     rejoin_interval: 0s
     left_ingesters_timeout: 5m0s
     leave_timeout: 20s
+    broadcast_timeout_for_local_updates_on_shutdown: 10s
     message_history_buffer_bytes: 0
     bind_addr: []
     bind_port: 7946

diff --git a/example/docker-compose/local/tempo.yaml b/example/docker-compose/local/tempo.yaml
@@ -17,6 +17,9 @@ query_frontend:
   search:
     duration_slo: 5s
     throughput_bytes_slo: 1.073741824e+09
+    metadata_slo:
+        duration_slo: 5s
+        throughput_bytes_slo: 1.073741824e+09
   trace_by_id:
     duration_slo: 5s
 

diff --git a/example/docker-compose/multi-tenant/tempo.yaml b/example/docker-compose/multi-tenant/tempo.yaml
@@ -10,6 +10,9 @@ query_frontend:
   search:
     duration_slo: 5s
     throughput_bytes_slo: 1.073741824e+09
+    metadata_slo:
+        duration_slo: 5s
+        throughput_bytes_slo: 1.073741824e+09
   trace_by_id:
     duration_slo: 5s
 

diff --git a/example/docker-compose/shared/tempo.yaml b/example/docker-compose/shared/tempo.yaml
@@ -7,6 +7,9 @@ query_frontend:
   search:
     duration_slo: 5s
     throughput_bytes_slo: 1.073741824e+09
+    metadata_slo:
+        duration_slo: 5s
+        throughput_bytes_slo: 1.073741824e+09
   trace_by_id:
     duration_slo: 5s