From 6736620df58f7bb12374132bdea780585113127e Mon Sep 17 00:00:00 2001
From: james-rms <james@foxglove.dev>
Date: Fri, 25 Oct 2024 14:38:22 +1100
Subject: [PATCH] client: actually stream messages in iter_messages (#114)

### Changelog
Fixed: `iter_messages` and `iter_decoded_messages` would unneccessarily
load the entire stream into RAM and sort them before yielding messages.
### Docs


### Description

Currently we create a stream from data platform and construct a
non-seeking reader around it to iterate through messages. However, (very
unfortunately) the default arguments to `iter_decoded_messages()`
require that the output stream be sorted, and the MCAP library does not
know that data platform streams are already sorted, so it loads the
entire stream into memory and sorts it before returning any messages.

<table><tr><th>Before</th><th>After</th></tr><tr><td>

<!--before content goes here-->

</td><td>

<!--after content goes here-->

</td></tr></table>

<!-- If necessary, link relevant Linear or Github issues. Use `Fixes:
foxglove/repo#1234` to auto-close the Github issue or Fixes: FG-### for
Linear isses. -->
---
 foxglove/client.py | 12 ++++++++++--
 setup.cfg          |  2 +-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/foxglove/client.py b/foxglove/client.py
index 14840e0..dd6ee9e 100644
--- a/foxglove/client.py
+++ b/foxglove/client.py
@@ -317,7 +317,12 @@ def get_messages(
         reader = make_reader(BytesIO(data), decoder_factories=decoder_factories)
         return [
             (channel.topic, message, decoded_message)
-            for _, channel, message, decoded_message in reader.iter_decoded_messages()
+            # messages from Foxglove are already in log-time order.
+            # specifying log_time_order=false allows us to skip a sort() in the MCAP library
+            # after all messages are loaded.
+            for _, channel, message, decoded_message in reader.iter_decoded_messages(
+                log_time_order=False,
+            )
         ]
 
     def iter_messages(
@@ -355,7 +360,10 @@ def iter_messages(
             # We deep-copy here as these factories might be mutated
             decoder_factories = copy.deepcopy(DEFAULT_DECODER_FACTORIES)
         reader = make_reader(response.raw, decoder_factories=decoder_factories)
-        return reader.iter_decoded_messages()
+        # messages from Foxglove are already in log-time order.
+        # specifying log_time_order=false allows us to skip a sort() in the MCAP library
+        # after all messages are loaded.
+        return reader.iter_decoded_messages(log_time_order=False)
 
     def download_recording_data(
         self,
diff --git a/setup.cfg b/setup.cfg
index a7f51d0..8c50d77 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = foxglove-client
-version = 0.15.2
+version = 0.15.3
 description = Client library for the Foxglove API.
 long_description = file: README.md
 long_description_content_type = text/markdown