Merge branch 'blakeblackshear:dev' into dev

frigate/events/cleanup.py

@@ -21,6 +21,9 @@ class EventCleanupType(str, Enum):
     snapshots = "snapshots"
 
 
+CHUNK_SIZE = 50
+
+
 class EventCleanup(threading.Thread):
     def __init__(
         self, config: FrigateConfig, stop_event: MpEvent, db: SqliteVecQueueDatabase
@@ -107,6 +110,7 @@ def expire(self, media_type: EventCleanupType) -> list[str]:
                 .namedtuples()
                 .iterator()
             )
+            logger.debug(f"{len(expired_events)} events can be expired")
             # delete the media from disk
             for expired in expired_events:
                 media_name = f"{expired.camera}-{expired.id}"
@@ -125,13 +129,34 @@ def expire(self, media_type: EventCleanupType) -> list[str]:
                     logger.warning(f"Unable to delete event images: {e}")
 
             # update the clips attribute for the db entry
-            update_query = Event.update(update_params).where(
+            query = Event.select(Event.id).where(
                 Event.camera.not_in(self.camera_keys),
                 Event.start_time < expire_after,
                 Event.label == event.label,
                 Event.retain_indefinitely == False,
             )
-            update_query.execute()
+
+            events_to_update = []
+
+            for batch in query.iterator():
+                events_to_update.extend([event.id for event in batch])
+                if len(events_to_update) >= CHUNK_SIZE:
+                    logger.debug(
+                        f"Updating {update_params} for {len(events_to_update)} events"
+                    )
+                    Event.update(update_params).where(
+                        Event.id << events_to_update
+                    ).execute()
+                    events_to_update = []
+
+            # Update any remaining events
+            if events_to_update:
+                logger.debug(
+                    f"Updating clips/snapshots attribute for {len(events_to_update)} events"
+                )
+                Event.update(update_params).where(
+                    Event.id << events_to_update
+                ).execute()
 
         events_to_update = []
 
@@ -196,7 +221,11 @@ def expire(self, media_type: EventCleanupType) -> list[str]:
                             logger.warning(f"Unable to delete event images: {e}")
 
         # update the clips attribute for the db entry
-        Event.update(update_params).where(Event.id << events_to_update).execute()
+        for i in range(0, len(events_to_update), CHUNK_SIZE):
+            batch = events_to_update[i : i + CHUNK_SIZE]
+            logger.debug(f"Updating {update_params} for {len(batch)} events")
+            Event.update(update_params).where(Event.id << batch).execute()
+
         return events_to_update
 
     def run(self) -> None:
@@ -222,10 +251,11 @@ def run(self) -> None:
                 .iterator()
             )
             events_to_delete = [e.id for e in events]
+            logger.debug(f"Found {len(events_to_delete)} events that can be expired")
             if len(events_to_delete) > 0:
-                chunk_size = 50
-                for i in range(0, len(events_to_delete), chunk_size):
-                    chunk = events_to_delete[i : i + chunk_size]
+                for i in range(0, len(events_to_delete), CHUNK_SIZE):
+                    chunk = events_to_delete[i : i + CHUNK_SIZE]
+                    logger.debug(f"Deleting {len(chunk)} events from the database")
                     Event.delete().where(Event.id << chunk).execute()
 
                     if self.config.semantic_search.enabled:

frigate/output/output.py

@@ -63,6 +63,7 @@ def receiveSignal(signalNumber, frame):
     birdseye: Optional[Birdseye] = None
     preview_recorders: dict[str, PreviewRecorder] = {}
     preview_write_times: dict[str, float] = {}
+    failed_frame_requests: dict[str, int] = {}
 
     move_preview_frames("cache")
 
@@ -99,7 +100,16 @@ def receiveSignal(signalNumber, frame):
 
         if frame is None:
             logger.debug(f"Failed to get frame {frame_id} from SHM")
+            failed_frame_requests[camera] = failed_frame_requests.get(camera, 0) + 1
+
+            if failed_frame_requests[camera] > config.cameras[camera].detect.fps:
+                logger.warning(
+                    f"Failed to retrieve many frames for {camera} from SHM, consider increasing SHM size if this continues."
+                )
+
             continue
+        else:
+            failed_frame_requests[camera] = 0
 
         # send camera frame to ffmpeg process if websockets are connected
         if any(

frigate/output/preview.py

@@ -154,6 +154,7 @@ def __init__(self, config: CameraConfig) -> None:
         self.start_time = 0
         self.last_output_time = 0
         self.output_frames = []
+
         if config.detect.width > config.detect.height:
             self.out_height = PREVIEW_HEIGHT
             self.out_width = (
@@ -274,7 +275,7 @@ def should_write_frame(
 
         return False
 
-    def write_frame_to_cache(self, frame_time: float, frame) -> None:
+    def write_frame_to_cache(self, frame_time: float, frame: np.ndarray) -> None:
         # resize yuv frame
         small_frame = np.zeros((self.out_height * 3 // 2, self.out_width), np.uint8)
         copy_yuv_to_position(
@@ -303,7 +304,7 @@ def write_data(
         current_tracked_objects: list[dict[str, any]],
         motion_boxes: list[list[int]],
         frame_time: float,
-        frame,
+        frame: np.ndarray,
     ) -> bool:
         # check for updated record config
         _, updated_record_config = self.config_subscriber.check_for_update()
@@ -332,6 +333,10 @@ def write_data(
                     self.output_frames,
                     self.requestor,
                 ).start()
+            else:
+                logger.debug(
+                    f"Not saving preview for {self.config.name} because there are no saved frames."
+                )
 
             # reset frame cache
             self.segment_end = (