Add speak detection model (#135)

* add speak detection and improve frame rate

* update run.py file

* update the camera_reader

* update speaking logi

* update the format

* update format

* fix format issue

Author: WenjinFu

.typos.toml

@@ -4,5 +4,8 @@ extend-ignore-re = []
 [default.extend-words]
 thr="thr"
 
+[default.extend-identifiers]
+solvePnP="solvePnP"
+
 [files]
 extend-exclude = [  ]

src/om1_vlm/anonymizationSys/face_recog_stream/camera_reader.py

@@ -2,6 +2,8 @@
 
 import logging
 import subprocess
+import threading
+import time
 from typing import Optional
 
 import cv2
@@ -11,7 +13,14 @@
 
 class CameraReader:
     """
-    Camera reader using OpenCV with V4L2 backend.
+    Camera reader using OpenCV with a V4L2 backend.
+
+    Capture runs in a BACKGROUND THREAD that continuously pulls the newest frame
+    and keeps only the latest one. ``read_frame()`` then returns that latest
+    frame without blocking. This mirrors what ``v4l2-ctl --stream-mmap`` does
+    (drain continuously) and avoids the classic OpenCV problem where a
+    synchronous ``cap.read()`` in the main loop serializes capture with
+    processing and throttles FPS far below what the sensor can deliver.
     """
 
     def __init__(
@@ -21,36 +30,37 @@ def __init__(
         height: int,
         fps: int,
         rotate_90_cw: bool = False,
+        threaded: bool = True,
     ):
-        """
-        Initialize the camera reader with the specified device and settings.
-
-        Parameters
-        ----------
-        device : str
-            Video device path (e.g., '/dev/video0').
-        width : int
-            Desired frame width.
-        height : int
-            Desired frame height.
-        fps : int
-            Desired frames per second.
-        rotate_90_cw : bool
-            Whether to rotate frames 90 degrees clockwise.
-        """
         self.device = device
         self.width = width
         self.height = height
         self.fps = fps
         self.rotate_90_cw = rotate_90_cw
+        self.threaded = threaded
 
         self.cap: Optional[cv2.VideoCapture] = None
+
+        # Latest-frame handoff between the grab thread and read_frame().
+        self._latest = None
+        self._latest_lock = threading.Lock()
+        self._frame_ready = threading.Event()
+        self._stop = threading.Event()
+        self._grab_thread: Optional[threading.Thread] = None
+
         self.open_camera()
 
+        if self.threaded:
+            self._grab_thread = threading.Thread(
+                target=self._grab_loop, name="camera-grab", daemon=True
+            )
+            self._grab_thread.start()
+            # Give the thread a moment to land the first frame so the first
+            # read_frame() isn't None.
+            self._frame_ready.wait(timeout=2.0)
+
     def open_camera(self):
-        """
-        Open the camera using the specified device and settings.
-        """
+        """Open the camera using the specified device and settings."""
         self.open_capture(self.device, self.width, self.height, int(self.fps))
         if self.cap is None or not self.cap.isOpened():
             raise RuntimeError(f"Failed to open camera on device {self.device}")
@@ -79,7 +89,6 @@ def open_capture(
             self.cap.set(cv2.CAP_PROP_FPS, fps)
             self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
             self.cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, 3)
-
             try:
                 subprocess.run(
                     [
@@ -104,10 +113,8 @@ def open_capture(
                     device,
                     (e.stderr or e.stdout or "").strip(),
                 )
-
             if not self.cap or not self.cap.isOpened():
                 raise RuntimeError(f"Failed to open camera device {device}")
-
             actual_width = int(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH))
             actual_height = int(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
             actual_fps = float(self.cap.get(cv2.CAP_PROP_FPS)) or float(fps)
@@ -119,11 +126,41 @@ def open_capture(
                 actual_fps,
             )
             self.width, self.height, self.fps = actual_width, actual_height, actual_fps
-
         except Exception as e:
             logging.error("Error opening camera %s: %s", device, e)
             self.cap = None
 
+    def _grab_loop(self):
+        """Continuously read the newest frame; keep only the latest."""
+        fail = 0
+        while not self._stop.is_set():
+            if self.cap is None or not self.cap.isOpened():
+                time.sleep(0.05)
+                try:
+                    self.cap = None
+                    self.open_capture(
+                        self.device, self.width, self.height, int(self.fps or 30)
+                    )
+                except Exception:
+                    pass
+                continue
+
+            ret, frame = self.cap.read()
+            if not ret or frame is None:
+                fail += 1
+                if fail % 50 == 0:
+                    logging.warning("camera grab failing (%d)", fail)
+                time.sleep(0.005)
+                continue
+            fail = 0
+
+            if self.rotate_90_cw:
+                frame = cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
+
+            with self._latest_lock:
+                self._latest = frame
+            self._frame_ready.set()
+
     def is_opened(self) -> bool:
         """
         Check if the camera is opened.
@@ -136,21 +173,25 @@ def is_opened(self) -> bool:
 
     def read_frame(self):
         """
-        Read a frame from the camera.
+        Return the most recent frame.
 
         Returns
         -------
-            The captured frame as a cv2.Mat object, or None if reading failed.
+        Threaded mode: non-blocking, returns the latest grabbed frame (or None
+        until the first one arrives). Non-threaded mode: the original synchronous
+        read, kept for compatibility.
         """
+        if self.threaded:
+            with self._latest_lock:
+                return None if self._latest is None else self._latest.copy()
+
         if not self.is_opened():
             logging.warning("Camera is not opened. Reopening...")
             self.open_capture(self.device, self.width, self.height, int(self.fps or 30))
-
         ret, frame = self.cap.read() if self.cap else (False, None)
         if not ret:
             logging.warning("Failed to read frame from camera.")
             return None
-
         if self.rotate_90_cw and frame is not None:
             frame = cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
         return frame
@@ -159,6 +200,9 @@ def release(self) -> None:
         """
         Release the camera resource.
         """
+        self._stop.set()
+        if self._grab_thread and self._grab_thread.is_alive():
+            self._grab_thread.join(timeout=1.0)
         if self.cap:
             self.cap.release()
             self.cap = None

src/om1_vlm/anonymizationSys/face_recog_stream/face_tracker.py

@@ -34,7 +34,10 @@
 import time
 from collections import Counter
 from dataclasses import dataclass, field
-from typing import Callable, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple
+
+if TYPE_CHECKING:
+    from .vvad_speaker import VVADScorer
 
 import cv2
 import numpy as np
@@ -255,6 +258,8 @@ def __init__(
         # Set as a plain attribute (like on_unknown_sample) by run.py, wired to
         # the auto-merge handler. None = feature off.
         self.on_identity_flip: Optional[Callable[[int, str, str], None]] = None
+        # Vision-only "who is speaking" scorer (set by run.py). Optional.
+        self.vvad: Optional["VVADScorer"] = None
 
         # Gallery (set via set_gallery). All three arrays are parallel —
         # row i of _gal_feats has name _gal_labels[i] and UUID _gal_uuids[i].
@@ -493,6 +498,23 @@ def update(
 
             self._active_ids.add(track_id)
 
+            # Feed the speaking-detector buffer. Pad the (tight) SCRFD box so the
+            # crop keeps the full mouth/chin context the VVAD model expects;
+            # clipping the mouth squashes the speaking score. Recognition still
+            # uses the original tight box below.
+            if self.vvad is not None:
+                bw, bh = x2 - x1, y2 - y1
+                px = int(bw * 0.20)
+                pyt = int(bh * 0.15)  # a little on top
+                pyb = int(bh * 0.30)  # more on bottom (mouth/chin)
+                cx1, cy1 = max(0, x1 - px), max(0, y1 - pyt)
+                cx2, cy2 = min(W, x2 + px), min(H, y2 + pyb)
+                self.vvad.push(
+                    track_id,
+                    frame[cy1:cy2, cx1:cx2],
+                    kps=track_det_map.get(track_id, {}).get("kps"),
+                )
+
             # Get or create identity state
             if track_id not in self._identities:
                 self._identities[track_id] = _TrackIdentity(
@@ -558,6 +580,9 @@ def update(
         # Cleanup tracks that BoTSORT stopped reporting
         self._cleanup_stale()
 
+        if self.vvad is not None:
+            self.vvad.evict(self._active_ids)
+
         # Build the by-area sorted face list used by /who and unknown capture
         self._current_faces = self._build_current_faces(results)