From 00371546a3c394bf1e4806b67e3d489854aa0b94 Mon Sep 17 00:00:00 2001 From: leccelecce <24962424+leccelecce@users.noreply.github.com> Date: Mon, 23 Dec 2024 14:05:34 +0000 Subject: [PATCH 1/8] GenAI: add ability to save JPGs sent to provider (#15643) * GenAI: add ability to save JPGs sent to provider * Remove mention from GenAI docs * Change config name to debug_save_thumbnails * Change folder structure to clips/genai-requests/{event_id}/{1.jpg} --- docs/docs/configuration/reference.md | 2 ++ frigate/config/camera/genai.py | 4 ++++ frigate/embeddings/maintainer.py | 31 +++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index b13b137d22..626e4f3c82 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -760,6 +760,8 @@ cameras: - cat # Optional: Restrict generation to objects that entered any of the listed zones (default: none, all zones qualify) required_zones: [] + # Optional: Save thumbnails sent to generative AI for review/debugging purposes (default: shown below) + debug_save_thumbnails: False # Optional ui: diff --git a/frigate/config/camera/genai.py b/frigate/config/camera/genai.py index 35c26eaf81..e6b3278361 100644 --- a/frigate/config/camera/genai.py +++ b/frigate/config/camera/genai.py @@ -38,6 +38,10 @@ class GenAICameraConfig(BaseModel): default_factory=list, title="List of required zones to be entered in order to run generative AI.", ) + debug_save_thumbnails: bool = Field( + default=False, + title="Save thumbnails sent to generative AI for debugging purposes.", + ) @field_validator("required_zones", mode="before") @classmethod diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index 4f81ec2d62..341a2e25dc 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -5,6 +5,7 @@ import os import threading from multiprocessing.synchronize import Event as MpEvent +from pathlib import Path from typing import Optional import cv2 @@ -217,6 +218,8 @@ def _process_finalized(self) -> None: _, buffer = cv2.imencode(".jpg", cropped_image) snapshot_image = buffer.tobytes() + num_thumbnails = len(self.tracked_events.get(event_id, [])) + embed_image = ( [snapshot_image] if event.has_snapshot and camera_config.genai.use_snapshot @@ -225,11 +228,37 @@ def _process_finalized(self) -> None: data["thumbnail"] for data in self.tracked_events[event_id] ] - if len(self.tracked_events.get(event_id, [])) > 0 + if num_thumbnails > 0 else [thumbnail] ) ) + if camera_config.genai.debug_save_thumbnails and num_thumbnails > 0: + logger.debug( + f"Saving {num_thumbnails} thumbnails for event {event.id}" + ) + + Path( + os.path.join(CLIPS_DIR, f"genai-requests/{event.id}") + ).mkdir(parents=True, exist_ok=True) + + for idx, data in enumerate(self.tracked_events[event_id], 1): + jpg_bytes: bytes = data["thumbnail"] + + if jpg_bytes is None: + logger.warning( + f"Unable to save thumbnail {idx} for {event.id}." + ) + else: + with open( + os.path.join( + CLIPS_DIR, + f"genai-requests/{event.id}/{idx}.jpg", + ), + "wb", + ) as j: + j.write(jpg_bytes) + # Generate the description. Call happens in a thread since it is network bound. threading.Thread( target=self._embed_description, From c48396c5c6e718ca021e01e84163107b2615f387 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 27 Dec 2024 08:36:21 -0600 Subject: [PATCH 2/8] Fix crash when streams are undefined in go2rtc config password cleaning (#15695) --- frigate/api/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frigate/api/app.py b/frigate/api/app.py index 57efdeb15c..7ab54ad86c 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -139,6 +139,8 @@ def config(request: Request): mode="json", warnings="none", exclude_none=True ) for stream_name, stream in go2rtc.get("streams", {}).items(): + if stream is None: + continue if isinstance(stream, str): cleaned = clean_camera_user_pass(stream) else: From 8114b541a8ac407ee6d6ff2cf114edc25ed3df38 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 27 Dec 2024 14:30:27 -0600 Subject: [PATCH 3/8] Sort camera group edit screen by ui config values (#15705) --- web/src/components/filter/CameraGroupSelector.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/web/src/components/filter/CameraGroupSelector.tsx b/web/src/components/filter/CameraGroupSelector.tsx index d1eb4b49ba..f99afcb836 100644 --- a/web/src/components/filter/CameraGroupSelector.tsx +++ b/web/src/components/filter/CameraGroupSelector.tsx @@ -755,7 +755,11 @@ export function CameraGroupEdit({ {[ ...(birdseyeConfig?.enabled ? ["birdseye"] : []), - ...Object.keys(config?.cameras ?? {}), + ...Object.keys(config?.cameras ?? {}).sort( + (a, b) => + (config?.cameras[a]?.ui?.order ?? 0) - + (config?.cameras[b]?.ui?.order ?? 0), + ), ].map((camera) => ( Date: Fri, 27 Dec 2024 18:38:43 -0600 Subject: [PATCH 4/8] Language consistency - use Explore instead of Search (#15709) --- web/src/components/settings/SearchSettings.tsx | 2 +- web/src/pages/Explore.tsx | 8 ++++---- web/src/pages/Settings.tsx | 4 ++-- web/src/views/settings/SearchSettingsView.tsx | 10 +++++----- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/web/src/components/settings/SearchSettings.tsx b/web/src/components/settings/SearchSettings.tsx index 17d319ec5b..788072ff19 100644 --- a/web/src/components/settings/SearchSettings.tsx +++ b/web/src/components/settings/SearchSettings.tsx @@ -46,7 +46,7 @@ export default function SearchSettings({ const trigger = ( - {search.has_snapshot && ( - - - - - - regenerateDescription("snapshot")} - > - Regenerate from Snapshot - - regenerateDescription("thumbnails")} - > - Regenerate from Thumbnails - - - - )} - - +
+ + {search.has_snapshot && ( + + + + + + regenerateDescription("snapshot")} + > + Regenerate from Snapshot + + regenerateDescription("thumbnails")} + > + Regenerate from Thumbnails + + + + )} +
+ )} + {(config?.cameras[search.camera].genai.enabled && search.end_time) || + (!config?.cameras[search.camera].genai.enabled && ( - - )} + ))} From e7ad38d8274ceb99b4f4412c1c4d6a858065d895 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Thu, 2 Jan 2025 10:04:16 -0600 Subject: [PATCH 8/8] Update model docs (#15779) --- docs/docs/configuration/object_detectors.md | 7 ++++++- docs/docs/configuration/reference.md | 14 ++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 5982da2ec8..e06baaad54 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -144,7 +144,9 @@ detectors: #### SSDLite MobileNet v2 -An OpenVINO model is provided in the container at `/openvino-model/ssdlite_mobilenet_v2.xml` and is used by this detector type by default. The model comes from Intel's Open Model Zoo [SSDLite MobileNet V2](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/ssdlite_mobilenet_v2) and is converted to an FP16 precision IR model. Use the model configuration shown below when using the OpenVINO detector with the default model. +An OpenVINO model is provided in the container at `/openvino-model/ssdlite_mobilenet_v2.xml` and is used by this detector type by default. The model comes from Intel's Open Model Zoo [SSDLite MobileNet V2](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/ssdlite_mobilenet_v2) and is converted to an FP16 precision IR model. + +Use the model configuration shown below when using the OpenVINO detector with the default OpenVINO model: ```yaml detectors: @@ -254,6 +256,7 @@ yolov4x-mish-640 yolov7-tiny-288 yolov7-tiny-416 yolov7-640 +yolov7-416 yolov7-320 yolov7x-640 yolov7x-320 @@ -282,6 +285,8 @@ The TensorRT detector can be selected by specifying `tensorrt` as the model type The TensorRT detector uses `.trt` model files that are located in `/config/model_cache/tensorrt` by default. These model path and dimensions used will depend on which model you have generated. +Use the config below to work with generated TRT models: + ```yaml detectors: tensorrt: diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index 626e4f3c82..7f31643690 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -117,25 +117,27 @@ auth: hash_iterations: 600000 # Optional: model modifications +# NOTE: The default values are for the EdgeTPU detector. +# Other detectors will require the model config to be set. model: - # Optional: path to the model (default: automatic based on detector) + # Required: path to the model (default: automatic based on detector) path: /edgetpu_model.tflite - # Optional: path to the labelmap (default: shown below) + # Required: path to the labelmap (default: shown below) labelmap_path: /labelmap.txt # Required: Object detection model input width (default: shown below) width: 320 # Required: Object detection model input height (default: shown below) height: 320 - # Optional: Object detection model input colorspace + # Required: Object detection model input colorspace # Valid values are rgb, bgr, or yuv. (default: shown below) input_pixel_format: rgb - # Optional: Object detection model input tensor format + # Required: Object detection model input tensor format # Valid values are nhwc or nchw (default: shown below) input_tensor: nhwc - # Optional: Object detection model type, currently only used with the OpenVINO detector + # Required: Object detection model type, currently only used with the OpenVINO detector # Valid values are ssd, yolox, yolonas (default: shown below) model_type: ssd - # Optional: Label name modifications. These are merged into the standard labelmap. + # Required: Label name modifications. These are merged into the standard labelmap. labelmap: 2: vehicle # Optional: Map of object labels to their attribute labels (default: depends on model)