Ensure genai thumbnails are always jpegs (#16939)

frigate/embeddings/maintainer.py

@@ -48,7 +48,11 @@
 from frigate.models import Event
 from frigate.types import TrackedObjectUpdateTypesEnum
 from frigate.util.builtin import serialize
-from frigate.util.image import SharedMemoryFrameManager, calculate_region
+from frigate.util.image import (
+    SharedMemoryFrameManager,
+    calculate_region,
+    ensure_jpeg_bytes,
+)
 from frigate.util.path import get_event_thumbnail_bytes
 
 from .embeddings import Embeddings
@@ -374,6 +378,9 @@ def _process_genai_description(self, event, camera_config, thumbnail) -> None:
 
         num_thumbnails = len(self.tracked_events.get(event.id, []))
 
+        # ensure we have a jpeg to pass to the model
+        thumbnail = ensure_jpeg_bytes(thumbnail)
+
         embed_image = (
             [snapshot_image]
             if event.has_snapshot and camera_config.genai.use_snapshot
@@ -503,6 +510,9 @@ def handle_regenerate_description(self, event_id: str, source: str) -> None:
 
         thumbnail = get_event_thumbnail_bytes(event)
 
+        # ensure we have a jpeg to pass to the model
+        thumbnail = ensure_jpeg_bytes(thumbnail)
+
         logger.debug(
             f"Trying {source} regeneration for {event}, has_snapshot: {event.has_snapshot}"
         )

frigate/util/image.py

@@ -975,3 +975,22 @@ def get_histogram(image, x_min, y_min, x_max, y_max):
         [image_bgr], [0, 1, 2], None, [8, 8, 8], [0, 256, 0, 256, 0, 256]
     )
     return cv2.normalize(hist, hist).flatten()
+
+
+def ensure_jpeg_bytes(image_data):
+    """Ensure image data is jpeg bytes for genai"""
+    try:
+        img_array = np.frombuffer(image_data, dtype=np.uint8)
+        img = cv2.imdecode(img_array, cv2.IMREAD_COLOR)
+
+        if img is None:
+            return image_data
+
+        success, encoded_img = cv2.imencode(".jpg", img)
+
+        if success:
+            return encoded_img.tobytes()
+    except Exception as e:
+        logger.warning(f"Error when converting thumbnail to jpeg for genai: {e}")
+
+    return image_data