From 70ba2f6e01ba0bce0f68aa4a18f448300d620ea0 Mon Sep 17 00:00:00 2001
From: james-rms <james@foxglove.dev>
Date: Fri, 7 Jun 2024 15:55:42 +1000
Subject: [PATCH] bump primary site to v0.0.41 (#83)

### Changelog
#### Inbox listener
- Changed: if a file is provided with an invalid device name or ID, the
import will fail immediately rather than being retried repeatedly.
- Added: MCAP files containing channels with schema ID 0 (no schema)
will be successfully imported.
- Added: Inbox listener will sort messages in files by their log time
while importing. See
https://docs.foxglove.dev/docs/primary-sites/self-hosting/inbox-listener-configuration
for configuration options relating to this change.
#### Stream server
- Fixed: when a stream server pod is sent SIGTERM, it waits
for`SHUTDOWN_TIMEOUT` seconds for any open requests to complete.
Previously when this timeout was exceeded, the stream server would exit
uncleanly and log an error. After this change, the stream server logs a
warning and exits cleanly.
- Added: stream server will serve MCAP channels with schema ID 0 (no
schema) successfully.

### Docs

https://github.com/foxglove/docs/pull/297

### Description

Updates revision to include features described in changelog. Also adds
configuration options to allow deployer to configure local disk for
their inbox listener to use.

### Testing
- [x] tested local disk configuration and use on my local sandbox
cluster, works as advertised
---
 charts/primary-site/Chart.yaml                |  6 ++++--
 .../templates/deployments/inbox-listener.yaml | 21 +++++++++++++++++++
 charts/primary-site/values.yaml               | 12 +++++++++++
 3 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/charts/primary-site/Chart.yaml b/charts/primary-site/Chart.yaml
index 4167eb3..c59ebea 100644
--- a/charts/primary-site/Chart.yaml
+++ b/charts/primary-site/Chart.yaml
@@ -13,6 +13,8 @@ type: application
 # 1.0.0-alpha.0
 # 1.0.0-alpha.1
 # 1.0.0
-version: "0.0.40"
+version: "0.0.41"
+
+appVersion: "610223e5f22ba95225f598df955e578e839f8d9a"
+
 
-appVersion: "e913c8f049a5ed252b4b54b0cbae08459fc5dd85"
diff --git a/charts/primary-site/templates/deployments/inbox-listener.yaml b/charts/primary-site/templates/deployments/inbox-listener.yaml
index 7e7d4e4..b864550 100644
--- a/charts/primary-site/templates/deployments/inbox-listener.yaml
+++ b/charts/primary-site/templates/deployments/inbox-listener.yaml
@@ -31,6 +31,11 @@ spec:
           secret:
             secretName: gcp-cloud-credential
             optional: true
+        {{- if .Values.inboxListener.deployment.localScratch.enabled }}
+        - name: local-scratch
+          emptyDir:
+            sizeLimit: {{ .Values.inboxListener.deployment.localScratch.capacityBytes }}
+        {{- end }}
       containers:
         - name: inbox-listener
           image: us-central1-docker.pkg.dev/foxglove-images/images/inbox-listener:{{ .Chart.AppVersion }}
@@ -38,12 +43,22 @@ spec:
             requests:
               cpu: {{ .Values.inboxListener.deployment.resources.requests.cpu }}
               memory: {{ .Values.inboxListener.deployment.resources.requests.memory }}
+              {{- if .Values.inboxListener.deployment.localScratch.enabled }}
+              ephemeral-storage: {{ .Values.inboxListener.deployment.localScratch.capacityBytes }}
+              {{- end}}
             limits:
               cpu: {{ .Values.inboxListener.deployment.resources.limits.cpu }}
               memory: {{ .Values.inboxListener.deployment.resources.limits.memory }}
+              {{- if .Values.inboxListener.deployment.localScratch.enabled }}
+              ephemeral-storage: {{ .Values.inboxListener.deployment.localScratch.capacityBytes }}
+              {{- end}}
           volumeMounts:
             - mountPath: /secrets
               name: cloud-credentials
+            {{- if .Values.inboxListener.deployment.localScratch.enabled }}
+            - mountPath: /local-scratch
+              name: local-scratch
+            {{- end }}
           ports:
             - name: metrics
               containerPort: 6001
@@ -102,6 +117,12 @@ spec:
             - name: {{ $item.name }}
               value: {{ $item.value | quote}}
             {{- end }}
+            {{- if .Values.inboxListener.deployment.localScratch.enabled }}
+            - name: LOCAL_SCRATCH_ROOT
+              value: "/local-scratch"
+            - name: LOCAL_SCRATCH_CAPACITY_BYTES
+              value: "{{ .Values.inboxListener.deployment.localScratch.capacityBytes }}"
+            {{- end }}
       {{- if .Values.inboxListener.deployment.serviceAccount.enabled }}
       serviceAccount: inbox-listener
       {{- end}}
diff --git a/charts/primary-site/values.yaml b/charts/primary-site/values.yaml
index 4bac0b6..8486280 100644
--- a/charts/primary-site/values.yaml
+++ b/charts/primary-site/values.yaml
@@ -74,6 +74,18 @@ inboxListener:
       # inbox listener.
       # - name: AWS_WRITER_PART_SIZE_BYTES
       #   value: 5242880
+      # SCRATCH_BUFFER_CAPACITY_BYTES: The maximum buffer size that will be used when reading
+      # indexed MCAP. During processing, some file data is read into memory before being written to
+      # the lake. If the message timestamps in a file are highly out-of-order, this limit may be
+      # exceeded. If the limit is exceeded, local disk or lake storage is used instead of memory.
+      # - name: SCRATCH_BUFFER_CAPACITY_BYTES
+      #   value: "1073741824"
+    localScratch:
+      # If your primary site is expected to process unindexed MCAP or ROS 1 BAG files, consider
+      # configuring your inbox listener deployment with local scratch storage. We recommend
+      # allocating at least three times as much capacity as your largest expected input file size.
+      enabled: false
+      capacityBytes: "107374182400"
 
     serviceAccount:
       enabled: false