feat: migrate to traces Grafana Cloud

chart/README.md

@@ -15,7 +15,7 @@ Helm chart to deploy all different types OTEL Collectors for infrastructure tele
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | apps | object | `{"container-logs":{"collector":"container-logs.yaml","enable":true,"spec":{"env":[{"name":"KUBE_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}}],"envFrom":[{"secretRef":{"name":"o2-cloud-secrets"}},{"configMapRef":{"name":"otel-common-config-map"}}],"mode":"daemonset","podAnnotations":{"<<":{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"},"atomi.cloud/module":"container-logs-collector"},"podSecurityContext":{"runAsNonRoot":false},"ports":[{"name":"zpages","port":55679,"targetPort":55679}],"resources":{"limits":{"cpu":1,"memory":"1Gi"},"requests":{"cpu":"250m","memory":"256Mi"}},"securityContext":{},"serviceAccount":"otel-container-logs-sa","volumeMounts":[{"mountPath":"/var/log/pods","name":"varlogpods","readOnly":true},{"mountPath":"/var/lib/docker/containers","name":"varlibdockercontainers","readOnly":true}],"volumes":[{"hostPath":{"path":"/var/log/pods"},"name":"varlogpods"},{"hostPath":{"path":"/var/lib/docker/containers"},"name":"varlibdockercontainers"}]}},"k8s-cluster":{"collector":"k8s-cluster.yaml","enable":true,"spec":{"envFrom":[{"secretRef":{"name":"o2-cloud-secrets"}},{"configMapRef":{"name":"otel-common-config-map"}}],"mode":"deployment","podAnnotations":{"<<":{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"},"atomi.cloud/module":"cluster-metrics-collector"},"podSecurityContext":{"<<":{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"ports":[{"name":"zpages","port":55679,"targetPort":55679}],"replicas":1,"resources":{"limits":{"cpu":"250m","memory":"1Gi"},"requests":{"cpu":"50m","memory":"256Mi"}},"securityContext":{"<<":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"serviceAccount":"otel-collector-k8scluster-sa"}},"k8s-events":{"collector":"k8s-events.yaml","enable":true,"spec":{"envFrom":[{"secretRef":{"name":"o2-cloud-secrets"}},{"configMapRef":{"name":"otel-common-config-map"}}],"mode":"deployment","podAnnotations":{"<<":{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"},"atomi.cloud/module":"cluster-events-collector"},"podSecurityContext":{"<<":{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"ports":[{"name":"zpages","port":55679,"targetPort":55679}],"replicas":1,"resources":{"limits":{"cpu":"250m","memory":"1Gi"},"requests":{"cpu":"50m","memory":"256Mi"}},"securityContext":{"<<":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"serviceAccount":"otel-collector-k8sevents-sa"}},"kubelet-stats":{"collector":"kubelet-stats-node-name.yaml","enable":true,"spec":{"env":[{"name":"K8S_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"KUBE_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}},{"name":"NODE_IP","valueFrom":{"fieldRef":{"fieldPath":"status.hostIP"}}}],"envFrom":[{"secretRef":{"name":"o2-cloud-secrets"}},{"configMapRef":{"name":"otel-common-config-map"}}],"mode":"daemonset","podAnnotations":{"<<":{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"},"atomi.cloud/module":"kubelet-stats-collector"},"podSecurityContext":{"<<":{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"ports":[{"name":"zpages","port":55679,"targetPort":55679}],"resources":{"limits":{"cpu":"250m","memory":"1Gi"},"requests":{"cpu":"50m","memory":"128Mi"}},"securityContext":{"<<":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"serviceAccount":"otel-collector-kubelet-sa"}},"otlp":{"collector":"otlp.yaml","enable":true,"spec":{"env":[{"name":"KUBE_NODE_NAME","valueFrom":{"fieldRef":{"fieldPath":"spec.nodeName"}}}],"envFrom":[{"secretRef":{"name":"o2-cloud-secrets"}},{"configMapRef":{"name":"otel-common-config-map"}}],"mode":"daemonset","podAnnotations":{"<<":{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"},"atomi.cloud/module":"otlp-collector"},"podSecurityContext":{"<<":{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"ports":[{"name":"zpages","port":55679,"targetPort":55679}],"resources":{"limits":{"cpu":"250m","memory":"1Gi"},"requests":{"cpu":"50m","memory":"128Mi"}},"securityContext":{"<<":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"serviceAccount":"otel-collector-otlp-sa"}},"target-allocator":{"collector":"ta.yaml","enable":true,"spec":{"envFrom":[{"secretRef":{"name":"o2-cloud-secrets"}},{"configMapRef":{"name":"otel-common-config-map"}}],"mode":"statefulset","podAnnotations":{"<<":{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"},"atomi.cloud/module":"target-allocator-collector"},"podSecurityContext":{"<<":{"fsGroup":1000,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"replicas":1,"resources":{"limits":{"cpu":"250m","memory":"1Gi"},"requests":{"cpu":"100m","memory":"256Mi"}},"securityContext":{"<<":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsGroup":1000,"runAsNonRoot":true,"runAsUser":1000}},"serviceAccount":"otel-collector-ta-sa","targetAllocator":{"enabled":true,"prometheusCR":{"enabled":true},"serviceAccount":"otel-collector-ta-sa"}}}}` | Dictionary of collectors to deploy. Key is the name of the collector, while the value is the configuration for the collector. This has 2 sub keys: `collector` which is the actual [collector configuration](https://opentelemetry.io/docs/collector/configuration/), and `spec`, which is the [operator's configuration](https://github.com/open-telemetry/opentelemetry-operator/blob/main/docs/api.md#opentelemetrycollectorspec) for the collector. |
-| auth | object | `{"external":{"enable":true,"policy":{"creation":"Owner","deletion":"Retain"},"secretStore":{"kind":"SecretStore","name":"doppler-silicon"}},"internal":{"enable":false,"loki":{"token":"sometoken","user":"someuser"},"o2":"sometoken"},"remote":{"loki":{"token":"MANUAL_LOKI_TOKEN","user":"MANUAL_LOKI_USER"},"o2":"MANUAL_O2_TOKEN"},"secretName":"o2-cloud-secrets"}` | Auth configuration for the collectors |
+| auth | object | `{"external":{"enable":true,"policy":{"creation":"Owner","deletion":"Retain"},"secretStore":{"kind":"SecretStore","name":"doppler-silicon"}},"internal":{"enable":false,"loki":{"token":"sometoken","user":"someuser"},"o2":"sometoken","tempo":{"token":"sometoken","user":"someuser"}},"remote":{"loki":{"token":"MANUAL_LOKI_TOKEN","user":"MANUAL_LOKI_USER"},"o2":"MANUAL_O2_TOKEN","tempo":{"token":"MANUAL_TEMPO_TOKEN","user":"MANUAL_TEMPO_USER"}},"secretName":"o2-cloud-secrets"}` | Auth configuration for the collectors |
 | auth.external | object | `{"enable":true,"policy":{"creation":"Owner","deletion":"Retain"},"secretStore":{"kind":"SecretStore","name":"doppler-silicon"}}` | Use external auth for the collectors |
 | auth.external.enable | bool | `true` | Enable external auth |
 | auth.external.policy | object | `{"creation":"Owner","deletion":"Retain"}` | External Secret Policy |
@@ -24,16 +24,21 @@ Helm chart to deploy all different types OTEL Collectors for infrastructure tele
 | auth.external.secretStore | object | `{"kind":"SecretStore","name":"doppler-silicon"}` | Secret Store to use for secrets |
 | auth.external.secretStore.kind | string | `"SecretStore"` | Kind of the secret store, either `ClusterSecretStore` or `SecretStore` |
 | auth.external.secretStore.name | string | `"doppler-silicon"` | Name of the secret store |
-| auth.internal | object | `{"enable":false,"loki":{"token":"sometoken","user":"someuser"},"o2":"sometoken"}` | Use internal auth for the collectors (hard coded password) |
+| auth.internal | object | `{"enable":false,"loki":{"token":"sometoken","user":"someuser"},"o2":"sometoken","tempo":{"token":"sometoken","user":"someuser"}}` | Use internal auth for the collectors (hard coded password) |
 | auth.internal.enable | bool | `false` | Enable internal auth |
 | auth.internal.loki | object | `{"token":"sometoken","user":"someuser"}` | Grafana Cloud Loki plaintext user |
 | auth.internal.loki.token | string | `"sometoken"` | Grafana Cloud Loki plaintext token |
 | auth.internal.loki.user | string | `"someuser"` | Grafana Cloud Loki plaintext user |
 | auth.internal.o2 | string | `"sometoken"` | OpenObserve plaintext token |
-| auth.remote | object | `{"loki":{"token":"MANUAL_LOKI_TOKEN","user":"MANUAL_LOKI_USER"},"o2":"MANUAL_O2_TOKEN"}` | Remote Tokens |
+| auth.internal.tempo | object | `{"token":"sometoken","user":"someuser"}` | Grafana Cloud Tempo plaintext user |
+| auth.internal.tempo.token | string | `"sometoken"` | Grafana Cloud Tempo plaintext token |
+| auth.internal.tempo.user | string | `"someuser"` | Grafana Cloud Tempo plaintext user |
+| auth.remote | object | `{"loki":{"token":"MANUAL_LOKI_TOKEN","user":"MANUAL_LOKI_USER"},"o2":"MANUAL_O2_TOKEN","tempo":{"token":"MANUAL_TEMPO_TOKEN","user":"MANUAL_TEMPO_USER"}}` | Remote Tokens |
 | auth.remote.loki.token | string | `"MANUAL_LOKI_TOKEN"` | Grafana Cloud Loki Token |
 | auth.remote.loki.user | string | `"MANUAL_LOKI_USER"` | Grafana Cloud Loki User |
 | auth.remote.o2 | string | `"MANUAL_O2_TOKEN"` | OpenObserve Token |
+| auth.remote.tempo.token | string | `"MANUAL_TEMPO_TOKEN"` | Grafana Cloud Tempo Token |
+| auth.remote.tempo.user | string | `"MANUAL_TEMPO_USER"` | Grafana Cloud Tempo User |
 | auth.secretName | string | `"o2-cloud-secrets"` | Name of the secret to use for the collector |
 | cluster | string | `"opal"` | Cluster the operators are deployed to |
 | configMapName | string | `"otel-common-config-map"` | Name of the common config map to propagate to all collectors |
@@ -79,6 +84,7 @@ Helm chart to deploy all different types OTEL Collectors for infrastructure tele
 | ta.serviceAccount.create | bool | `true` | Enable creation of the service account |
 | ta.serviceAccount.name | string | `"otel-collector-ta-sa"` | Name of the service account |
 | tags | object | `{"argocd.argoproj.io/compare-options":"IgnoreExtraneous","atomi.cloud/layer":"1","atomi.cloud/platform":"sulfoxide","atomi.cloud/service":"silicon"}` | Kubernetes labels and annotations, following Service Tree |
+| tempoEndpoint | string | `"https://tempo-prod-14-prod-ap-southeast-1.grafana.net/tempo"` | Grafana Cloud Tempo Endpoint |
 
 ----------------------------------------------
 Autogenerated from chart metadata using [helm-docs v1.11.1](https://github.com/norwoodj/helm-docs/releases/v1.11.1)

chart/collectors/otlp.yaml

@@ -102,18 +102,26 @@ exporters:
     endpoint: ${env:LOKI_ENDPOINT}
     auth:
       authenticator: basicauth/loki
+  otlphttp:
+    endpoint: ${env:TEMPO_ENDPOINT}
+    auth:
+      authenticator: basicauth/tempo
 
 extensions:
   health_check:
   pprof:
   zpages:
     endpoint: 0.0.0.0:55679
+  basicauth/tempo:
+    client_auth:
+      username: ${env:TEMPO_USER}
+      password: ${env:TEMPO_PASSWORD}
   basicauth/loki:
     client_auth:
       username: ${env:LOKI_USER}
       password: ${env:LOKI_TOKEN}
 service:
-  extensions: [ health_check, pprof, zpages, basicauth/loki ]
+  extensions: [ health_check, pprof, zpages, basicauth/loki, basicauth/tempo ]
   pipelines:
     logs:
       receivers: [ otlp ]
@@ -122,7 +130,7 @@ service:
     traces:
       receivers: [ otlp ]
       processors: [ k8sattributes, resource, attributes, batch ]
-      exporters: [ otlphttp/openobserve ]
+      exporters: [ otlphttp ]
     metrics:
       receivers: [ otlp ]
       processors: [ k8sattributes, resource, attributes, batch ]

chart/templates/otel-common-config.yaml

@@ -10,4 +10,5 @@ data:
   TA_ENDPOINT: "{{ .Values.taEndpoint }}"
   O2_ENDPOINT: "{{ .Values.o2Endpoint }}"
   LOKI_ENDPOINT: "{{ .Values.lokiEndpoint }}"
+  TEMPO_ENDPOINT: "{{ .Values.tempoEndpoint }}"
 

chart/templates/secrets.yaml

@@ -25,6 +25,12 @@ spec:
     - secretKey: LOKI_TOKEN
       remoteRef:
         key: {{ .Values.auth.remote.loki.token }}
+    - secretKey: TEMPO_USER
+      remoteRef:
+        key: {{ .Values.auth.remote.tempo.user }}
+    - secretKey: TEMPO_TOKEN
+      remoteRef:
+        key: {{ .Values.auth.remote.tempo.token }}
 ---
 {{- end }}
 {{- if .Values.auth.internal.enable }}
@@ -39,4 +45,6 @@ data:
   O2_AUTH: {{ b64enc .Values.auth.internal.o2 }}
   LOKI_USER: {{ b64enc .Values.auth.internal.loki.user }}
   LOKI_TOKEN: {{ b64enc .Values.auth.internal.loki.token }}
+  TEMPO_USER: {{ b64enc .Values.auth.internal.tempo.user }}
+  TEMPO_TOKEN: {{ b64enc .Values.auth.internal.tempo.token }}
 {{- end }}

chart/values.yaml

@@ -23,6 +23,9 @@ o2Endpoint: "https://api.openobserve.ai/api/atomicloud_MwvsSHPiOT9uFdn/"
 # -- Grafana Cloud Loki Endpoint
 lokiEndpoint: "https://logs-prod-020.grafana.net/loki/api/v1/push"
 
+# -- Grafana Cloud Tempo Endpoint
+tempoEndpoint: "https://tempo-prod-14-prod-ap-southeast-1.grafana.net/tempo"
+
 # -- Name of the common config map to propagate to all collectors
 configMapName: &configMapName otel-common-config-map
 
@@ -132,6 +135,11 @@ auth:
       user: MANUAL_LOKI_USER
       # -- Grafana Cloud Loki Token
       token: MANUAL_LOKI_TOKEN
+    tempo:
+      # -- Grafana Cloud Tempo User
+      user: MANUAL_TEMPO_USER
+      # -- Grafana Cloud Tempo Token
+      token: MANUAL_TEMPO_TOKEN
 
   # -- Use internal auth for the collectors (hard coded password)
   internal:
@@ -145,6 +153,12 @@ auth:
       user: someuser
       # -- Grafana Cloud Loki plaintext token
       token: sometoken
+    # -- Grafana Cloud Tempo plaintext user
+    tempo:
+      # -- Grafana Cloud Tempo plaintext user
+      user: someuser
+      # -- Grafana Cloud Tempo plaintext token
+      token: sometoken
   # -- Use external auth for the collectors
   external:
     # -- Enable external auth