Merge branch 'blakeblackshear:dev' into 0.16_ObjectSpeed_Hybrid_AVXFix

docs/docs/configuration/object_detectors.md

@@ -144,7 +144,9 @@ detectors:
 
 #### SSDLite MobileNet v2
 
-An OpenVINO model is provided in the container at `/openvino-model/ssdlite_mobilenet_v2.xml` and is used by this detector type by default. The model comes from Intel's Open Model Zoo [SSDLite MobileNet V2](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/ssdlite_mobilenet_v2) and is converted to an FP16 precision IR model. Use the model configuration shown below when using the OpenVINO detector with the default model.
+An OpenVINO model is provided in the container at `/openvino-model/ssdlite_mobilenet_v2.xml` and is used by this detector type by default. The model comes from Intel's Open Model Zoo [SSDLite MobileNet V2](https://github.com/openvinotoolkit/open_model_zoo/tree/master/models/public/ssdlite_mobilenet_v2) and is converted to an FP16 precision IR model. 
+
+Use the model configuration shown below when using the OpenVINO detector with the default OpenVINO model:
 
 ```yaml
 detectors:
@@ -254,6 +256,7 @@ yolov4x-mish-640
 yolov7-tiny-288
 yolov7-tiny-416
 yolov7-640
+yolov7-416
 yolov7-320
 yolov7x-640
 yolov7x-320
@@ -282,6 +285,8 @@ The TensorRT detector can be selected by specifying `tensorrt` as the model type
 
 The TensorRT detector uses `.trt` model files that are located in `/config/model_cache/tensorrt` by default. These model path and dimensions used will depend on which model you have generated.
 
+Use the config below to work with generated TRT models:
+
 ```yaml
 detectors:
   tensorrt:

docs/docs/configuration/reference.md

@@ -117,25 +117,27 @@ auth:
   hash_iterations: 600000
 
 # Optional: model modifications
+# NOTE: The default values are for the EdgeTPU detector.
+# Other detectors will require the model config to be set.
 model:
-  # Optional: path to the model (default: automatic based on detector)
+  # Required: path to the model (default: automatic based on detector)
   path: /edgetpu_model.tflite
-  # Optional: path to the labelmap (default: shown below)
+  # Required: path to the labelmap (default: shown below)
   labelmap_path: /labelmap.txt
   # Required: Object detection model input width (default: shown below)
   width: 320
   # Required: Object detection model input height (default: shown below)
   height: 320
-  # Optional: Object detection model input colorspace
+  # Required: Object detection model input colorspace
   # Valid values are rgb, bgr, or yuv. (default: shown below)
   input_pixel_format: rgb
-  # Optional: Object detection model input tensor format
+  # Required: Object detection model input tensor format
   # Valid values are nhwc or nchw (default: shown below)
   input_tensor: nhwc
-  # Optional: Object detection model type, currently only used with the OpenVINO detector
+  # Required: Object detection model type, currently only used with the OpenVINO detector
   # Valid values are ssd, yolox, yolonas (default: shown below)
   model_type: ssd
-  # Optional: Label name modifications. These are merged into the standard labelmap.
+  # Required: Label name modifications. These are merged into the standard labelmap.
   labelmap:
     2: vehicle
   # Optional: Map of object labels to their attribute labels (default: depends on model)
@@ -772,6 +774,8 @@ cameras:
         - cat
       # Optional: Restrict generation to objects that entered any of the listed zones (default: none, all zones qualify)
       required_zones: []
+      # Optional: Save thumbnails sent to generative AI for review/debugging purposes (default: shown below)
+      debug_save_thumbnails: False
 
 # Optional
 ui:

frigate/api/app.py

@@ -142,6 +142,8 @@ def config(request: Request):
         mode="json", warnings="none", exclude_none=True
     )
     for stream_name, stream in go2rtc.get("streams", {}).items():
+        if stream is None:
+            continue
         if isinstance(stream, str):
             cleaned = clean_camera_user_pass(stream)
         else:

frigate/config/camera/genai.py

@@ -38,6 +38,10 @@ class GenAICameraConfig(BaseModel):
         default_factory=list,
         title="List of required zones to be entered in order to run generative AI.",
     )
+    debug_save_thumbnails: bool = Field(
+        default=False,
+        title="Save thumbnails sent to generative AI for debugging purposes.",
+    )
 
     @field_validator("required_zones", mode="before")
     @classmethod

frigate/embeddings/maintainer.py

@@ -8,6 +8,7 @@
 import string
 import threading
 from multiprocessing.synchronize import Event as MpEvent
+from pathlib import Path
 from typing import Optional
 
 import cv2
@@ -324,6 +325,8 @@ def _process_finalized(self) -> None:
                             _, buffer = cv2.imencode(".jpg", cropped_image)
                             snapshot_image = buffer.tobytes()
 
+                    num_thumbnails = len(self.tracked_events.get(event_id, []))
+
                     embed_image = (
                         [snapshot_image]
                         if event.has_snapshot and camera_config.genai.use_snapshot
@@ -332,11 +335,37 @@ def _process_finalized(self) -> None:
                                 data["thumbnail"]
                                 for data in self.tracked_events[event_id]
                             ]
-                            if len(self.tracked_events.get(event_id, [])) > 0
+                            if num_thumbnails > 0
                             else [thumbnail]
                         )
                     )
 
+                    if camera_config.genai.debug_save_thumbnails and num_thumbnails > 0:
+                        logger.debug(
+                            f"Saving {num_thumbnails} thumbnails for event {event.id}"
+                        )
+
+                        Path(
+                            os.path.join(CLIPS_DIR, f"genai-requests/{event.id}")
+                        ).mkdir(parents=True, exist_ok=True)
+
+                        for idx, data in enumerate(self.tracked_events[event_id], 1):
+                            jpg_bytes: bytes = data["thumbnail"]
+
+                            if jpg_bytes is None:
+                                logger.warning(
+                                    f"Unable to save thumbnail {idx} for {event.id}."
+                                )
+                            else:
+                                with open(
+                                    os.path.join(
+                                        CLIPS_DIR,
+                                        f"genai-requests/{event.id}/{idx}.jpg",
+                                    ),
+                                    "wb",
+                                ) as j:
+                                    j.write(jpg_bytes)
+
                     # Generate the description. Call happens in a thread since it is network bound.
                     threading.Thread(
                         target=self._embed_description,

frigate/events/cleanup.py

@@ -121,8 +121,8 @@ def expire_snapshots(self) -> list[str]:
 
             events_to_update = []
 
-            for batch in query.iterator():
-                events_to_update.extend([event.id for event in batch])
+            for event in query.iterator():
+                events_to_update.append(event.id)
                 if len(events_to_update) >= CHUNK_SIZE:
                     logger.debug(
                         f"Updating {update_params} for {len(events_to_update)} events"
@@ -257,7 +257,7 @@ def expire_clips(self) -> list[str]:
         events_to_update = []
 
         for event in query.iterator():
-            events_to_update.append(event)
+            events_to_update.append(event.id)
 
             if len(events_to_update) >= CHUNK_SIZE:
                 logger.debug(

web/src/components/filter/CameraGroupSelector.tsx

@@ -825,7 +825,11 @@ export function CameraGroupEdit({
                 <FormMessage />
                 {[
                   ...(birdseyeConfig?.enabled ? ["birdseye"] : []),
-                  ...Object.keys(config?.cameras ?? {}),
+                  ...Object.keys(config?.cameras ?? {}).sort(
+                    (a, b) =>
+                      (config?.cameras[a]?.ui?.order ?? 0) -
+                      (config?.cameras[b]?.ui?.order ?? 0),
+                  ),
                 ].map((camera) => (
                   <FormControl key={camera}>
                     <div className="flex items-center justify-between gap-1">

web/src/components/overlay/detail/ObjectLifecycle.tsx

@@ -477,7 +477,10 @@ export default function ObjectLifecycle({
                           </p>
                           {Array.isArray(item.data.box) &&
                           item.data.box.length >= 4
-                            ? (item.data.box[2] / item.data.box[3]).toFixed(2)
+                            ? (
+                                aspectRatio *
+                                (item.data.box[2] / item.data.box[3])
+                              ).toFixed(2)
                             : "N/A"}
                         </div>
                       </div>

web/src/components/overlay/detail/SearchDetailDialog.tsx

@@ -550,54 +550,54 @@ function ObjectDetailsTab({
 
         <div className="flex w-full flex-row justify-end gap-2">
           {config?.cameras[search.camera].genai.enabled && search.end_time && (
-            <>
-              <div className="flex items-start">
-                <Button
-                  className="rounded-r-none border-r-0"
-                  aria-label="Regenerate tracked object description"
-                  onClick={() => regenerateDescription("thumbnails")}
-                >
-                  Regenerate
-                </Button>
-                {search.has_snapshot && (
-                  <DropdownMenu>
-                    <DropdownMenuTrigger asChild>
-                      <Button
-                        className="rounded-l-none border-l-0 px-2"
-                        aria-label="Expand regeneration menu"
-                      >
-                        <FaChevronDown className="size-3" />
-                      </Button>
-                    </DropdownMenuTrigger>
-                    <DropdownMenuContent>
-                      <DropdownMenuItem
-                        className="cursor-pointer"
-                        aria-label="Regenerate from snapshot"
-                        onClick={() => regenerateDescription("snapshot")}
-                      >
-                        Regenerate from Snapshot
-                      </DropdownMenuItem>
-                      <DropdownMenuItem
-                        className="cursor-pointer"
-                        aria-label="Regenerate from thumbnails"
-                        onClick={() => regenerateDescription("thumbnails")}
-                      >
-                        Regenerate from Thumbnails
-                      </DropdownMenuItem>
-                    </DropdownMenuContent>
-                  </DropdownMenu>
-                )}
-              </div>
-
+            <div className="flex items-start">
+              <Button
+                className="rounded-r-none border-r-0"
+                aria-label="Regenerate tracked object description"
+                onClick={() => regenerateDescription("thumbnails")}
+              >
+                Regenerate
+              </Button>
+              {search.has_snapshot && (
+                <DropdownMenu>
+                  <DropdownMenuTrigger asChild>
+                    <Button
+                      className="rounded-l-none border-l-0 px-2"
+                      aria-label="Expand regeneration menu"
+                    >
+                      <FaChevronDown className="size-3" />
+                    </Button>
+                  </DropdownMenuTrigger>
+                  <DropdownMenuContent>
+                    <DropdownMenuItem
+                      className="cursor-pointer"
+                      aria-label="Regenerate from snapshot"
+                      onClick={() => regenerateDescription("snapshot")}
+                    >
+                      Regenerate from Snapshot
+                    </DropdownMenuItem>
+                    <DropdownMenuItem
+                      className="cursor-pointer"
+                      aria-label="Regenerate from thumbnails"
+                      onClick={() => regenerateDescription("thumbnails")}
+                    >
+                      Regenerate from Thumbnails
+                    </DropdownMenuItem>
+                  </DropdownMenuContent>
+                </DropdownMenu>
+              )}
+            </div>
+          )}
+          {(config?.cameras[search.camera].genai.enabled && search.end_time) ||
+            (!config?.cameras[search.camera].genai.enabled && (
               <Button
                 variant="select"
                 aria-label="Save"
                 onClick={updateDescription}
               >
                 Save
               </Button>
-            </>
-          )}
+            ))}
         </div>
       </div>
     </div>

web/src/components/settings/SearchSettings.tsx

@@ -46,7 +46,7 @@ export default function SearchSettings({
   const trigger = (
     <Button
       className="flex items-center gap-2"
-      aria-label="Search Settings"
+      aria-label="Explore Settings"
       size="sm"
     >
       <FaCog className="text-secondary-foreground" />

web/src/pages/Explore.tsx

@@ -328,12 +328,12 @@ export default function Explore() {
           <div className="flex max-w-96 flex-col items-center justify-center space-y-3 rounded-lg bg-background/50 p-5">
             <div className="my-5 flex flex-col items-center gap-2 text-xl">
               <TbExclamationCircle className="mb-3 size-10" />
-              <div>Search Unavailable</div>
+              <div>Explore is Unavailable</div>
             </div>
             {embeddingsReindexing && allModelsLoaded && (
               <>
                 <div className="text-center text-primary-variant">
-                  Search can be used after tracked object embeddings have
+                  Explore can be used after tracked object embeddings have
                   finished reindexing.
                 </div>
                 <div className="pt-5 text-center">
@@ -384,8 +384,8 @@ export default function Explore() {
               <>
                 <div className="text-center text-primary-variant">
                   Frigate is downloading the necessary embeddings models to
-                  support semantic searching. This may take several minutes
-                  depending on the speed of your network connection.
+                  support the Semantic Search feature. This may take several
+                  minutes depending on the speed of your network connection.
                 </div>
                 <div className="flex w-96 flex-col gap-2 py-5">
                   <div className="flex flex-row items-center justify-center gap-2">

web/src/pages/Settings.tsx

@@ -40,7 +40,7 @@ import UiSettingsView from "@/views/settings/UiSettingsView";
 
 const allSettingsViews = [
   "UI settings",
-  "search settings",
+  "explore settings",
   "camera settings",
   "masks / zones",
   "motion tuner",
@@ -175,7 +175,7 @@ export default function Settings() {
       </div>
       <div className="mt-2 flex h-full w-full flex-col items-start md:h-dvh md:pb-24">
         {page == "UI settings" && <UiSettingsView />}
-        {page == "search settings" && (
+        {page == "explore settings" && (
           <SearchSettingsView setUnsavedChanges={setUnsavedChanges} />
         )}
         {page == "debug" && (

web/src/views/settings/SearchSettingsView.tsx

@@ -91,7 +91,7 @@ export default function SearchSettingsView({
       )
       .then((res) => {
         if (res.status === 200) {
-          toast.success("Search settings have been saved.", {
+          toast.success("Explore settings have been saved.", {
             position: "top-center",
           });
           setChangedValue(false);
@@ -128,7 +128,7 @@ export default function SearchSettingsView({
     if (changedValue) {
       addMessage(
         "search_settings",
-        `Unsaved search settings changes`,
+        `Unsaved Explore settings changes`,
         undefined,
         "search_settings",
       );
@@ -140,7 +140,7 @@ export default function SearchSettingsView({
   }, [changedValue]);
 
   useEffect(() => {
-    document.title = "Search Settings - Frigate";
+    document.title = "Explore Settings - Frigate";
   }, []);
 
   if (!config) {
@@ -152,7 +152,7 @@ export default function SearchSettingsView({
       <Toaster position="top-center" closeButton={true} />
       <div className="scrollbar-container order-last mb-10 mt-2 flex h-full w-full flex-col overflow-y-auto rounded-lg border-[1px] border-secondary-foreground bg-background_alt p-2 md:order-none md:mb-0 md:mr-2 md:mt-0">
         <Heading as="h3" className="my-2">
-          Search Settings
+          Explore Settings
         </Heading>
         <Separator className="my-2 flex bg-secondary" />
         <Heading as="h4" className="my-2">
@@ -221,7 +221,7 @@ export default function SearchSettingsView({
               <div className="text-md">Model Size</div>
               <div className="space-y-1 text-sm text-muted-foreground">
                 <p>
-                  The size of the model used for semantic search embeddings.
+                  The size of the model used for Semantic Search embeddings.
                 </p>
                 <ul className="list-disc pl-5 text-sm">
                   <li>