add KEDA based autoscaling support

foxglove · Aug 22, 2024 · 47a40ba · 47a40ba
1 parent 4468985
commit 47a40ba
Show file tree

Hide file tree

Showing 2 changed files with 156 additions and 0 deletions.
diff --git a/charts/primary-site/templates/deployments/inbox-listener.yaml b/charts/primary-site/templates/deployments/inbox-listener.yaml
@@ -1,3 +1,149 @@
+{{- if .Values.inboxListener.autoscaling.enabled }}
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+  name: keda-inbox-listener-auth
+spec:
+  secretTargetRef:
+    - parameter: token
+      name: foxglove-site-token
+      key: FOXGLOVE_SITE_TOKEN
+---
+apiVersion: keda.sh/v1alpha1
+kind: ScaledJob
+metadata:
+  name: inbox-listener-scaledjob
+spec:
+  minReplicaCount: {{ .Values.inboxListener.autoscaling.minReplicas }}
+  maxReplicaCount: {{ .Values.inboxListener.autoscaling.maxReplicas }}
+  successfulJobsHistoryLimit: 50
+  failedJobsHistoryLimit: 100
+  pollingInterval: 30
+  jobTargetRef:
+    parallelism: 1
+    activeDeadlineSeconds: 86400
+    template:
+      metadata:
+        labels:
+          app: inbox-listener
+          {{- range $key, $value := .Values.inboxListener.deployment.podLabels }}
+          {{ $key }}: {{ $value | quote }}
+          {{- end }}
+        annotations:
+          {{- range $key, $value := .Values.inboxListener.deployment.podAnnotations }}
+          {{ $key }}: {{ $value | quote }}
+          {{- end }}
+      spec:
+        volumes:
+          - name: cloud-credentials
+            secret:
+              secretName: gcp-cloud-credential
+              optional: true
+              {{- if .Values.inboxListener.deployment.localScratch.enabled }}
+          - name: local-scratch
+            emptyDir:
+              sizeLimit: {{ .Values.inboxListener.deployment.localScratch.capacityBytes }}
+              {{- end }}
+            {{- if .Values.inboxListener.deployment.nodeSelectors }}
+        nodeSelector:
+              {{- range $key, $value := .Values.inboxListener.deployment.nodeSelectors }}
+              {{ $key }}: {{ $value | quote }}
+              {{- end }}
+            {{- end}}
+        {{- if .Values.inboxListener.deployment.serviceAccount.enabled }}
+        serviceAccount: inbox-listener
+        {{- end}}
+        containers:
+          - name: inbox-listener
+            image: us-central1-docker.pkg.dev/foxglove-images/images/inbox-listener:{{ .Chart.AppVersion }}
+            resources:
+              requests:
+                cpu: {{ .Values.inboxListener.deployment.resources.requests.cpu }}
+                memory: {{ .Values.inboxListener.deployment.resources.requests.memory }}
+                  {{- if .Values.inboxListener.deployment.localScratch.enabled }}
+                ephemeral-storage: {{ .Values.inboxListener.deployment.localScratch.capacityBytes }}
+                  {{- end}}
+              limits:
+                cpu: {{ .Values.inboxListener.deployment.resources.limits.cpu }}
+                memory: {{ .Values.inboxListener.deployment.resources.limits.memory }}
+                  {{- if .Values.inboxListener.deployment.localScratch.enabled }}
+                ephemeral-storage: {{ .Values.inboxListener.deployment.localScratch.capacityBytes }}
+                  {{- end}}
+            volumeMounts:
+              - mountPath: /secrets
+                name: cloud-credentials
+                {{- if .Values.inboxListener.deployment.localScratch.enabled }}
+              - mountPath: /local-scratch
+                name: local-scratch
+                {{- end }}
+            ports:
+              - name: metrics
+                containerPort: 6001
+            envFrom:
+              - secretRef:
+                  name: cloud-credentials
+                  optional: true
+              - secretRef:
+                  name: foxglove-site-token
+                  optional: false
+                {{- range $k := .Values.globals.secrets }}
+              - secretRef:
+                  name: {{ $k }}
+                {{- end }}
+            env:
+                {{ with lookup "v1" "Secret" .Release.Namespace "gcp-cloud-credential" }}
+              ## The lookup is required here. The pod may have access to GCP through other means, but
+              ## the credentials in this env var take precedence, even if it's empty. An empty variable
+              ## essentially blocks GCP access.
+              - name: GOOGLE_APPLICATION_CREDENTIALS
+                value: /secrets/credentials.json
+                {{ end }}
+              - name: FOXGLOVE_API_URL
+                value: "{{ .Values.globals.foxgloveApiUrl }}"
+              - name: MODE
+                value: self-managed
+              - name: INBOX_STORAGE_PROVIDER
+                value: "{{ .Values.globals.inbox.storageProvider }}"
+              - name: STORAGE_INBOX_BUCKET_NAME
+                value: "{{ .Values.globals.inbox.bucketName }}"
+              - name: LAKE_STORAGE_PROVIDER
+                value: "{{ .Values.globals.lake.storageProvider }}"
+              - name: STORAGE_LAKE_BUCKET_NAME
+                value: "{{ .Values.globals.lake.bucketName }}"
+              - name: STORAGE_AZURE_STORAGE_ACCOUNT_NAME
+                value: "{{ .Values.globals.azure.storageAccountName }}"
+              - name: STORAGE_AZURE_SERVICE_URL
+                value: "{{ .Values.globals.azure.serviceUrl }}"
+              - name: AWS_REGION
+                value: "{{ .Values.globals.aws.region }}"
+              - name: AWS_SDK_LOAD_CONFIG
+                value: "true"
+              - name: PROMETHEUS_METRICS_NAMESPACE
+                value: "{{ .Values.inboxListener.deployment.metrics.namespace }}"
+              - name: PROMETHEUS_METRICS_SUBSYSTEM
+                value: "{{ .Values.inboxListener.deployment.metrics.subsystem }}"
+                {{- range $item := .Values.inboxListener.deployment.env }}
+              - name: {{ $item.name }}
+                value: {{ $item.value | quote}}
+                {{- end }}
+              - name: MAX_WAIT_FOR_WORK
+                value: 30s
+                {{- if .Values.inboxListener.deployment.localScratch.enabled }}
+              - name: LOCAL_SCRATCH_ROOT
+                value: "/local-scratch"
+              - name: LOCAL_SCRATCH_CAPACITY_BYTES
+                value: "{{ .Values.inboxListener.deployment.localScratch.capacityBytes }}"
+                {{- end }}
+  triggers:
+    - type: metrics-api
+      metadata:
+        targetValue: "2"
+        url: "{{ .Values.globals.foxgloveApiUrl }}/internal/platform/v1/pending-imports-stats"
+        valueLocation: "unleased"
+        authMode: "bearer"
+      authenticationRef:
+        name: keda-inbox-listener-auth
+{{- else }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -132,3 +278,4 @@ spec:
         {{ $key }}: {{ $value | quote }}
         {{- end }}
       {{- end}}
+{{- end }}
diff --git a/charts/primary-site/values.yaml b/charts/primary-site/values.yaml
@@ -97,6 +97,15 @@ inboxListener:
       ## annotations:
       ##   eks.amazonaws.com/role-arn: arn:aws:iam::xxxxxxxxxxxx:role/foxglove-inbox-listener-sa-role
 
+  # To enable the autoscaling built into this chart, you must install KEDA first
+  # helm repo add kedacore https://kedacore.github.io/charts
+  # helm repo update
+  # helm install keda kedacore/keda --namespace keda --create-namespace
+  autoscaling:
+    enabled: false
+    minReplicas: 1
+    maxReplicas: 10
+
 streamService:
   service:
     annotations: {}