Merge pull request #3989 from lsst-sqre/tickets/DM-47389/mobu-metrics…

…-for-real

DM-47389: mobu app metrics events

applications/mobu/Chart.yaml

@@ -5,4 +5,4 @@ description: "Continuous integration testing"
 home: https://mobu.lsst.io/
 sources:
   - "https://github.com/lsst-sqre/mobu"
-appVersion: 13.0.1
+appVersion: 13.2.0

applications/mobu/README.md

@@ -17,6 +17,11 @@ Continuous integration testing
 | config.githubCiApp | string | disabled. | Configuration for the GitHub refresh app integration. See https://mobu.lsst.io/operations/github_ci_app.html#add-phalanx-configuration |
 | config.githubRefreshApp | string | disabled. | Configuration for the GitHub refresh app integration. See https://mobu.lsst.io/operations/github_refresh_app.html#add-phalanx-configuration |
 | config.logLevel | string | `"INFO"` | Log level. Set to 'DEBUG' to include the output from all flocks in the main mobu log. |
+| config.metrics.application | string | `"mobu"` | Name under which to log metrics. Generally there is no reason to change this. |
+| config.metrics.enabled | bool | `false` | Whether to enable sending metrics |
+| config.metrics.events.topicPrefix | string | `"lsst.square.metrics.events"` | Topic prefix for events. It may sometimes be useful to change this in development environments. |
+| config.metrics.schemaManager.registryUrl | string | Sasquatch in the local cluster | URL of the Confluent-compatible schema registry server |
+| config.metrics.schemaManager.suffix | string | `""` | Suffix to add to all registered subjects. This is sometimes useful for experimentation during development. |
 | config.pathPrefix | string | `"/mobu"` | Prefix for mobu's API routes. |
 | config.profile | string | `"production"` | One of 'production' or 'development'. 'production' configures structured JSON logging, and 'development' configures unstructured human readable logging. |
 | config.slackAlerts | bool | `true` | Whether to send alerts and status to Slack. |

applications/mobu/templates/deployment.yaml

@@ -64,6 +64,24 @@ spec:
                   name: {{ template "mobu.fullname" . }}-secret
                   key: "github-ci-app-webhook-secret"
             {{- end}}
+            {{- if .Values.config.metrics.enabled }}
+            - name: "KAFKA_BOOTSTRAP_SERVERS"
+              valueFrom:
+                secretKeyRef:
+                  name: "mobu-kafka"
+                  key: "bootstrapServers"
+            - name: "KAFKA_SECURITY_PROTOCOL"
+              valueFrom:
+                secretKeyRef:
+                  name: "mobu-kafka"
+                  key: "securityProtocol"
+            - name: "KAFKA_CLIENT_CERT_PATH"
+              value: "/etc/mobu-kafka/user.crt"
+            - name: "KAFKA_CLIENT_KEY_PATH"
+              value: "/etc/mobu-kafka/user.key"
+            - name: "KAFKA_CLUSTER_CA_PATH"
+              value: "/etc/mobu-kafka/ca.crt"
+            {{- end }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
           ports:
@@ -91,6 +109,20 @@ spec:
               readOnly: true
             - name: "tmp"
               mountPath: "/tmp"
+            {{- if .Values.config.metrics.enabled }}
+            - name: "kafka"
+              mountPath: "/etc/mobu-kafka/ca.crt"
+              readOnly: true
+              subPath: "ssl.truststore.crt"
+            - name: "kafka"
+              mountPath: "/etc/mobu-kafka/user.crt"
+              readOnly: true
+              subPath: "ssl.keystore.crt"
+            - name: "kafka"
+              mountPath: "/etc/mobu-kafka/user.key"
+              readOnly: true
+              subPath: "ssl.keystore.key"
+            {{- end }}
       securityContext:
         runAsNonRoot: true
         runAsUser: 1000
@@ -101,6 +133,11 @@ spec:
             name: "mobu"
         - name: "tmp"
           emptyDir: {}
+        {{- if .Values.config.metrics.enabled }}
+        - name: "kafka"
+          secret:
+            secretName: "mobu-kafka"
+        {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

applications/mobu/templates/kafka-access.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.config.metrics.enabled -}}
+apiVersion: access.strimzi.io/v1alpha1
+kind: KafkaAccess
+metadata:
+  name: "mobu-kafka"
+spec:
+  kafka:
+    name: "sasquatch"
+    namespace: "sasquatch"
+    listener: "tls"
+  user:
+    kind: "KafkaUser"
+    apiGroup: "kafka.strimzi.io"
+    name: "app-metrics-mobu"
+    namespace: "sasquatch"
+{{- end }}

applications/mobu/tests/github_ci_app_enabled_test.yaml

@@ -64,6 +64,14 @@ tests:
               - bot-mobu-ci-user-2
             githubRefreshApp: null
             logLevel: INFO
+            metrics:
+              application: mobu
+              enabled: false
+              events:
+                topicPrefix: lsst.square.metrics.events
+              schemaManager:
+                registryUrl: http://sasquatch-schema-registry.sasquatch.svc.cluster.local:8081
+                suffix: ""
             pathPrefix: /mobu
             profile: production
             slackAlerts: true

applications/mobu/tests/github_disabled_test.yaml

@@ -17,6 +17,14 @@ tests:
             githubCiApp: null
             githubRefreshApp: null
             logLevel: INFO
+            metrics:
+              application: mobu
+              enabled: false
+              events:
+                topicPrefix: lsst.square.metrics.events
+              schemaManager:
+                registryUrl: http://sasquatch-schema-registry.sasquatch.svc.cluster.local:8081
+                suffix: ""
             pathPrefix: /mobu
             profile: production
             slackAlerts: true

applications/mobu/tests/github_refresh_app_enabled_test.yaml

@@ -42,6 +42,14 @@ tests:
               - org1
               - org2
             logLevel: INFO
+            metrics:
+              application: mobu
+              enabled: false
+              events:
+                topicPrefix: lsst.square.metrics.events
+              schemaManager:
+                registryUrl: http://sasquatch-schema-registry.sasquatch.svc.cluster.local:8081
+                suffix: ""
             pathPrefix: /mobu
             profile: production
             slackAlerts: true

applications/mobu/values-idfdev.yaml

@@ -1,6 +1,8 @@
 config:
   logLevel: DEBUG
   profile: development
+  metrics:
+    enabled: true
   githubRefreshApp:
     acceptedGithubOrgs:
       - lsst-sqre

applications/mobu/values.yaml

@@ -52,6 +52,28 @@ config:
   # -- Prefix for mobu's API routes.
   pathPrefix: "/mobu"
 
+  metrics:
+    # -- Whether to enable sending metrics
+    enabled: false
+
+    # -- Name under which to log metrics. Generally there is no reason to
+    # change this.
+    application: "mobu"
+
+    events:
+      # -- Topic prefix for events. It may sometimes be useful to change this
+      # in development environments.
+      topicPrefix: "lsst.square.metrics.events"
+
+    schemaManager:
+      # -- URL of the Confluent-compatible schema registry server
+      # @default -- Sasquatch in the local cluster
+      registryUrl: "http://sasquatch-schema-registry.sasquatch.svc.cluster.local:8081"
+
+      # -- Suffix to add to all registered subjects. This is sometimes useful
+      # for experimentation during development.
+      suffix: ""
+
 # -- Resource limits and requests for the mobu frontend pod
 # @default -- See `values.yaml`
 resources:

applications/sasquatch/charts/app-metrics/values.yaml

@@ -13,7 +13,13 @@ globalAppConfig:
       - "username"
   mobu:
     influxTags:
-      - "type"
+      - "sync"
+      - "success"
+      - "flock"
+      - "business"
+      - "username"
+      - "notebook"
+      - "repo"
   wobbly:
     influxTags:
       - "service"