diff --git a/.gitignore b/.gitignore index 2c7af04..9fdd4dc 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ __pycache__ *.engine *.onnx venv/ +.venv/ diff --git a/src/om1_vlm/anonymizationSys/face_recog_stream/__init__.py b/src/om1_vlm/anonymizationSys/face_recog_stream/__init__.py index 662d8f2..a75ffc0 100644 --- a/src/om1_vlm/anonymizationSys/face_recog_stream/__init__.py +++ b/src/om1_vlm/anonymizationSys/face_recog_stream/__init__.py @@ -10,6 +10,7 @@ "scrfd", "gallery", "io", + "selfie_logic", "http_api", "who_tracker", "yolo_pose", diff --git a/src/om1_vlm/anonymizationSys/face_recog_stream/auto_enroller.py b/src/om1_vlm/anonymizationSys/face_recog_stream/auto_enroller.py new file mode 100644 index 0000000..09b941e --- /dev/null +++ b/src/om1_vlm/anonymizationSys/face_recog_stream/auto_enroller.py @@ -0,0 +1,675 @@ +""" +Auto-enrollment: silently build a UUID for a face that keeps showing up unknown. + +How it works +------------ +1. Per recognition pass, face_tracker hands us (track_id, aligned_crop, + embedding, sim_top1) for tracks that came back ``"uncertain"``. We do + no GPU work here — embedding is already computed upstream. + +2. For each track, we maintain a small buffer of recent samples (default + max_buffer=5). Samples must pass a per-frame quality bar (frontality, + sharpness, face size) — bad frames are dropped immediately. + +3. When a track's buffer hits ``n_samples_required`` AND the pairwise + cosine similarity between buffered samples is tight enough + (``min_pairwise_sim``, default 0.6) — meaning they all look like the + same person, not a flicker between two visitors — we commit a new + UUID with name="" (anonymous). The LLM can name it later via + ``/set_name``. + +4. Tracks that disappear (no more frames) get their buffer dropped after + ``stale_track_sec``. Tracks that get identified by face_tracker also + get dropped (no need to enroll them). + +Why this exists +--------------- +Right now, enrollment requires the LLM to ask "what's your name?" and +call /selfie. For drop-in visitors who never speak, we'd never enroll +them. With auto-enroll, the gallery quietly grows; the LLM only has to +*name* the identities, not create them. Naming is then a snappy O(1) +metadata write instead of a multi-frame collection loop. + +Threading model +--------------- +Same as gallery.py — all methods should be called from the main thread +(via the recognition callback inside face_tracker.update). No internal +locking. Maintains its own per-track buffer; doesn't touch shared state +except by calling gallery methods. +""" + +from __future__ import annotations + +import logging +import time +from collections import deque +from dataclasses import dataclass, field +from typing import Callable, Deque, Dict, List, Optional, Tuple + +import numpy as np + +from . import selfie_logic as sl + +log = logging.getLogger(__name__) + + +# Tier name from selfie_logic — duplicated here as a string constant so this +# module doesn't import face_tracker / selfie_logic (cleaner dependency graph). +_TIER_UNCERTAIN = "uncertain" + + +# --------------------------------------------------------------------------- +# Per-track buffer +# --------------------------------------------------------------------------- + + +@dataclass +class _TrackBuffer: + """Rolling buffer of recent samples for one unknown track. + + We keep aligned crops AND their embeddings — embeddings for the + tightness/novelty checks, crops for the eventual commit to the gallery. + + Timestamps + ---------- + first_seen : first time we created a buffer for this track. Drives the + ``min_unknown_sec`` gate. + last_seen : most recent ``observe()`` call for this track (whether or + not the sample was buffered). Drives ``gc_stale`` — if no observe + has fired in ``stale_track_sec``, the track is gone and we drop + the buffer. + + Note: there's no ``last_buffered`` because sample-spacing is controlled + by embedding-based novelty (not time). See AutoEnroller.observe() and + the ``novelty_thr`` parameter for the rationale. + """ + + track_id: int + crops: Deque[np.ndarray] = field(default_factory=deque) + embeddings: Deque[np.ndarray] = field(default_factory=deque) + first_seen: float = 0.0 + last_seen: float = 0.0 + # True after we've successfully enrolled this track — prevents double + # enrollment if more samples arrive before face_tracker picks up the new + # gallery centroid and starts identifying this track as "confident". + committed_uuid: Optional[str] = None + + +# --------------------------------------------------------------------------- +# AutoEnroller +# --------------------------------------------------------------------------- + + +class AutoEnroller: + """Watches unknown tracks and silently enrolls them when confident. + + Parameters + ---------- + gallery : UUIDGallery + Where new identities are created. Must be writable from the calling + thread (i.e. the main thread when this is wired via run_job_sync). + n_samples_required : int, default 3 + How many quality samples must accumulate before we even consider + enrolling. Lower → faster enrollment, more risk of bad identities. + max_buffer : int, default 5 + Maximum samples kept per track. When full, oldest is dropped. Should + be ≥ n_samples_required. + min_pairwise_sim : float, default 0.55 + Required pairwise cosine similarity (worst pair) across the buffer + before enrollment. Filters out tracks that "flicker" between two + people (BoTSORT ID switch on similar-looking visitors). Conservative + starting value; tune up if you see misenrollments. + min_unknown_sec : float, default 1.5 + Track must have been visible/unknown for at least this long before + enrollment fires. Keeps us from enrolling people who just walked + through the frame. + min_face_pixels : int, default 80 + Minimum bbox short side. Below this, samples are skipped — they're + too small to be reliable. + capture_interval_sec : float, default 0.3 + Minimum time between buffered samples for the same track. Prevents + the buffer from filling with near-duplicates from consecutive frames. + stale_track_sec : float, default 5.0 + Drop a track's buffer if no observe() in this long. Saves memory on + tracks that walked off. + merge_thr : float, default 0.50 + Before creating a NEW UUID, check whether the buffer's mean embedding + is close enough to any existing UUID's centroid. If max cosine sim + across the gallery >= ``merge_thr``, append the samples to that + existing UUID instead of creating a new one. + + Why this exists: ``SIM_THR`` is the per-frame recognition floor — + below it, a face is labeled "unknown" even if it's a known person + in bad lighting (top1=0.52 but margin=0.20 → "uncertain" because + 0.52 < SIM_THR=0.55, but actually it's clearly wendy). Without this + merge step, AutoEnroller would create a duplicate UUID for wendy + every time she walks into a different lighting condition. + + The buffer mean is more reliable than any single frame's embedding + (averaging suppresses noise), so this check uses the WHOLE buffer's + centroid against the gallery — typically higher than the per-frame + top1, often crossing into the "definitely this known person" zone. + + Set lower than SIM_THR. Setting to 0 disables the merge-on-commit + path (always creates new UUIDs, original behavior). + source_tag : str, default "auto_enroll" + ``source`` value to write into metadata.json for traceability. + on_committed : callable, optional + Optional callback ``on_committed(uuid, track_id, sample_count)`` + fired immediately after a successful enrollment. Useful for the + main loop to refresh face_tracker's gallery snapshot. + + Why no per-sample novelty/consistency check + ------------------------------------------- + Earlier iterations had two embedding-based gates here: + - novelty (reject if sim_to_mean > 0.97): tried to avoid buffering + near-duplicate frames + - consistency (reject if sim_to_mean < 0.50): tried to catch + BoTSORT track-id swaps mid-buffer + + Problem with novelty: a person standing naturally still produces + sim ≈ 0.95-0.99 between consecutive frames. Setting novelty_thr + permissively (0.97) starves the buffer in the still case; setting + it loosely (0.99) makes the gate near-meaningless. The "window" + (sim < novelty_thr AND sim > consistency_thr) often excludes + perfectly valid same-person frames. + + Problem with consistency vs tightness: the per-sample consistency + check is mostly redundant with the at-commit tightness check + (``min_pairwise_sim``). Both reject "buffer contains samples of + multiple people". + + Resolution: skip both per-sample checks. Accept all quality samples + into the buffer; rely on tightness at commit time. Within-pose + averaging from 3+ similar samples still yields a more robust + centroid than any single sample. Cross-pose diversity comes from + repeat encounters via the ``merge_thr`` path, not from one session. + """ + + def __init__( + self, + gallery, + *, + n_samples_required: int = 3, + max_buffer: int = 5, + min_pairwise_sim: float = 0.55, + min_unknown_sec: float = 1.0, + min_face_pixels: int = 30, + stale_track_sec: float = 5.0, + merge_thr: float = 0.50, + source_tag: str = "auto_enroll", + on_committed: Optional[Callable[[str, int, int], None]] = None, + min_conf: float = 0.0, + min_frontality: float = 0.0, + ): + self.gallery = gallery + self.n_required = int(n_samples_required) + self.max_buffer = max(int(max_buffer), self.n_required) + self.min_pairwise_sim = float(min_pairwise_sim) + self.min_unknown_sec = float(min_unknown_sec) + self.min_face_pixels = int(min_face_pixels) + self.stale_track_sec = float(stale_track_sec) + self.merge_thr = float(merge_thr) + self.source_tag = str(source_tag) + self.on_committed = on_committed + # Dedicated auto-enroll quality gates, independent of the /selfie + # thresholds. 0.0 = gate disabled. + self.min_conf = float(min_conf) + self.min_frontality = float(min_frontality) + + # track_id → buffer + self._buffers: Dict[int, _TrackBuffer] = {} + + # Stats (for /who debug) + self.enrolled_count = 0 + self.merged_count = 0 # NEW: tracks merge-on-commit decisions + self.rejected_count = 0 + + # ------------------------------------------------------------------ + # Main entry points + # ------------------------------------------------------------------ + + def observe( + self, + track_id: int, + aligned_crop: np.ndarray, + embedding: np.ndarray, + bbox: Tuple[int, int, int, int], + tier: str, + kps: Optional[np.ndarray] = None, + conf: float = 1.0, + ) -> Optional[str]: + """Feed one frame's recognition result for one track. + + Called from face_tracker._run_recognition_batch for EVERY track that + ran recognition this batch (not just unknown ones — passing + identified tracks lets us drop their buffers if they were previously + unknown). + + Parameters + ---------- + track_id : int + BoTSORT track id. Stable across frames for the same person. + aligned_crop : ndarray (112, 112, 3) BGR + Same warped crop that was fed to AdaFace. Reused for storage. + embedding : ndarray (512,) float32, L2-normalized + AdaFace output for ``aligned_crop``. + bbox : (x1, y1, x2, y2) + Face bbox in image coordinates. Used for the size gate. + tier : str + Recognition tier from selfie_logic.recognize_from_sims. + We only buffer samples when tier is "uncertain"; tracks that + became identified get their buffer cleared. + + Returns + ------- + Optional[str] + If this observation triggered an enrollment, returns the new + UUID. Otherwise None. Most calls return None. + """ + now = time.time() + + # Track became identified or tentative → drop any buffer, no enroll + if tier != _TIER_UNCERTAIN: + self._buffers.pop(track_id, None) + return None + + # Quality / size gate happens FIRST — don't even create a buffer + # entry for samples that fail the bar. Avoids leaking empty entries. + x1, y1, x2, y2 = bbox + if min(x2 - x1, y2 - y1) < self.min_face_pixels: + return None + + # Dedicated auto-enroll quality gate (independent of /selfie): require + # a confident, frontal detection before buffering, so only clean + # frontal faces enter the gallery. Does NOT affect what the robot can + # SEE/track — the global DETECTION_CONFIDENCE stays low; this only + # gates enrollment. 0.0 thresholds = gate off. + if self.min_conf > 0.0 and conf < self.min_conf: + return None + if self.min_frontality > 0.0 and kps is not None: + if sl.frontality(kps) < self.min_frontality: + return None + + # Get or create buffer + buf = self._buffers.get(track_id) + if buf is None: + buf = _TrackBuffer( + track_id=track_id, + first_seen=now, + last_seen=now, + ) + self._buffers[track_id] = buf + + # Always mark the track as recently seen — keeps gc_stale honest + # even when the sample below gets rate-limited away. + buf.last_seen = now + + # Already enrolled this track — wait for face_tracker to start + # identifying it as "confident". Don't enroll again. + if buf.committed_uuid is not None: + return None + + # No per-sample novelty/consistency check. We accept every quality + # sample and rely on min_pairwise_sim (the tightness check at commit + # time) to catch BoTSORT track-id swaps. See the class docstring + # for the rationale. + + # Buffer this sample (drop oldest if at capacity) + buf.crops.append(aligned_crop.copy()) + buf.embeddings.append(embedding.copy()) + if len(buf.crops) > self.max_buffer: + buf.crops.popleft() + buf.embeddings.popleft() + + # Eligibility checks + if len(buf.embeddings) < self.n_required: + return None + if (now - buf.first_seen) < self.min_unknown_sec: + return None + + # Pairwise tightness check + worst = self._min_pairwise_sim(list(buf.embeddings)) + if worst < self.min_pairwise_sim: + log.debug( + "auto-enroll track=%d skipped: pairwise=%.3f < %.3f", + track_id, + worst, + self.min_pairwise_sim, + ) + self.rejected_count += 1 + # Don't drop the buffer — maybe later samples improve cohesion. + # But cap how much we wait by trimming oldest if we exceeded buffer. + return None + + # All gates passed → commit + uuid = self._commit(buf) + return uuid + + def force_commit(self, track_id: int, name: str = "") -> Optional[str]: + """Commit a track's buffer immediately, even if gates aren't satisfied. + + Used by /selfie when the LLM has just learned the person's name but + auto-enroll hasn't fired yet for this track. The selfie collection + loop itself produces fresh samples; this method drains whatever + auto-enroll has already buffered. + + Returns the UUID, or None if the track has no buffer or only 0 + samples (nothing to commit). + """ + buf = self._buffers.get(track_id) + if buf is None or not buf.crops: + return None + if buf.committed_uuid is not None: + # Already committed — caller can use that UUID + return buf.committed_uuid + uuid = self._commit(buf, name=name, forced=True) + return uuid + + def drop_track(self, track_id: int) -> None: + """Explicitly drop a track's buffer (e.g. when track was identified + as a known person by a different path). + """ + self._buffers.pop(track_id, None) + + def gc_stale(self) -> int: + """Remove buffers for tracks that haven't been observed recently. + + Returns the number of buffers dropped. Called once per frame (cheap) + from the main loop, OR every few seconds from a slower timer. + """ + now = time.time() + cutoff = now - self.stale_track_sec + stale = [tid for tid, buf in self._buffers.items() if buf.last_seen < cutoff] + for tid in stale: + self._buffers.pop(tid, None) + return len(stale) + + def try_commit_pending(self) -> List[str]: + """Scan all active buffers and commit any that pass all gates. + + WHY THIS EXISTS + --------------- + AutoEnroller's commit logic in ``observe()`` is event-driven — it + only runs when face_tracker calls observe() with a new sample. + But after face_tracker finalizes a track as ``status="unknown"``, + it stops calling observe() until re-identify fires (≥1s later). + + Without this method, a buffer that accumulated enough samples + BEFORE the track went "unknown" would just sit there waiting: + - sample_count gate: PASSED (≥ n_required) + - tightness gate: PASSED (samples are consistent) + - time gate: MAY pass during the dead zone + + but no observe() is being called to trigger the gate check. So + the UUID creation gets delayed by up to re_identify_interval + seconds, even though every condition has been met. + + This method, called periodically from the main loop, polls all + buffers and commits any that pass all gates RIGHT NOW. Solves + the "system knows it's a stranger but hasn't enrolled them yet" + latency. + + Idempotent — already-committed buffers are skipped. Returns the + list of UUIDs created during this call (usually empty, sometimes + 1+ if multiple unknown tracks ripened at once). + + Cheap: typically iterates over a handful of buffers, each gate + check is O(1) or O(n²) on small embedding lists (n ≤ max_buffer). + Safe to call every frame, though every ~1s is enough. + """ + if not self._buffers: + return [] + now = time.time() + committed: List[str] = [] + for track_id, buf in list(self._buffers.items()): + # Skip already-committed buffers + if buf.committed_uuid is not None: + continue + # Sample count gate + if len(buf.embeddings) < self.n_required: + continue + # Time gate + if (now - buf.first_seen) < self.min_unknown_sec: + continue + # Tightness gate + worst = self._min_pairwise_sim(list(buf.embeddings)) + if worst < self.min_pairwise_sim: + # Tight enough samples haven't accumulated; leave for now. + # If the track is gone, gc_stale will drop the buffer later. + continue + # All gates pass → commit + uuid = self._commit(buf) + if uuid: + committed.append(uuid) + if committed: + log.info( + "try_commit_pending: committed %d pending buffer(s)", + len(committed), + ) + return committed + + # ------------------------------------------------------------------ + # Introspection + # ------------------------------------------------------------------ + + def snapshot(self) -> dict: + """Diagnostic snapshot for /who or a debug endpoint.""" + return { + "enrolled_total": self.enrolled_count, + "merged_total": self.merged_count, + "rejected_total": self.rejected_count, + "active_buffers": len(self._buffers), + "buffers": { + tid: { + "samples_buffered": len(buf.crops), + "first_seen_ago": round(time.time() - buf.first_seen, 1), + "committed_uuid": buf.committed_uuid, + } + for tid, buf in self._buffers.items() + }, + } + + # ------------------------------------------------------------------ + # Internals + # ------------------------------------------------------------------ + + def _commit( + self, + buf: _TrackBuffer, + name: str = "", + forced: bool = False, + ) -> str: + """Commit a buffered track into the gallery. + + Decision path: + 1. Compute the buffer's mean embedding (more reliable than any + single frame — averaging suppresses noise). + 2. Compare against ALL existing centroids. If the best match's + cosine sim >= ``merge_thr``, append samples to that UUID + (merge path). This catches the "known person in bad lighting + showed up as 'uncertain' per-frame, but their buffer mean is + clearly them" case. + 3. Otherwise, create a brand-new UUID. + + Forced commits (from /set_name) ALWAYS create a new UUID with the + given name — the LLM has named this specific track, so honor the + intent even if the embedding is close to an existing identity. + (If the user has a twin in the gallery, forced commit is the right + choice; the LLM/operator made the call.) + + Sets ``buf.committed_uuid`` so further observe() calls for this + track don't try to enroll again. + """ + # 1. Try merging into an existing UUID (unless forced — see docstring) + target_uuid: Optional[str] = None + merged_into_name: str = "" + merged_sim: float = 0.0 + if not forced and self.merge_thr > 0.0: + target_uuid, merged_into_name, merged_sim = self._find_merge_target(buf) + + try: + if target_uuid is None: + # 2a. Create new UUID + uuid = self.gallery.create( + name=name, + source=("auto_enroll_forced" if forced else self.source_tag), + first_track_id=int(buf.track_id), + buffered_samples=len(buf.crops), + ) + else: + # 2b. Merge into existing UUID + uuid = target_uuid + + saved = self.gallery.add_samples( + uuid, + list(buf.crops), + embeddings=list(buf.embeddings), + ) + except Exception as e: + log.exception("auto-enroll commit failed for track %d: %s", buf.track_id, e) + return "" + + buf.committed_uuid = uuid + if target_uuid is None: + self.enrolled_count += 1 + log.info( + "auto-enroll track=%d → NEW uuid=%s (samples=%d, forced=%s, name=%r)", + buf.track_id, + uuid, + len(saved), + forced, + name, + ) + else: + self.merged_count += 1 + log.info( + "auto-enroll track=%d → MERGED into uuid=%s name=%r (samples=%d, sim=%.3f)", + buf.track_id, + uuid, + merged_into_name, + len(saved), + merged_sim, + ) + + if self.on_committed is not None: + try: + self.on_committed(uuid, buf.track_id, len(saved)) + except Exception as e: + log.warning("on_committed callback raised: %s", e) + return uuid + + def _find_merge_target( + self, + buf: _TrackBuffer, + ) -> Tuple[Optional[str], str, float]: + """Check if buf.embeddings' mean matches an existing ANON UUID strongly + enough to fold the new samples in. Returns ``(uuid, name, sim)`` of the + best ANON match if sim >= merge_thr, else ``(None, "", 0.0)``. + + IMPORTANT: only ANON UUIDs (name="") are valid merge targets. + Named identities (sean, alice, ...) are NEVER auto-merged into, + because face embedding sim is unreliable enough that a passing + stranger could mistakenly contaminate a named person's identity. + Concrete example: + sean has UUID A (name="sean") in gallery. + stranger walks in, looks vaguely like sean. + buffer mean sim to A's centroid = 0.52 (>= merge_thr=0.50). + WITHOUT this guard: stranger's samples get folded into sean's UUID. + → sean's centroid is polluted, sean's recognition + suffers for everyone going forward. + WITH this guard: A is named → skipped. Create new anon UUID + instead. LLM/operator can later confirm via + /gallery/find_similar + /gallery/merge if + they're really the same person. + + Why use the buffer mean instead of per-frame embeddings: + The per-frame recognition pass already saw each individual + embedding and concluded "uncertain" (otherwise this track + wouldn't have buffered). But the MEAN of N tight samples is a + cleaner signal than any single frame — noise averages out, + identity signal stays. So the mean often crosses the merge + threshold even when no single frame did. + + Example of legitimate auto-merge: + anon UUID X was created last session (face seen but not named). + Same person walks in again, gets buffered to a new track. + Buffer mean sim against X's centroid = 0.65. + X has name="" (anon) → merge into X (samples folded in). + """ + if not buf.embeddings: + return None, "", 0.0 + try: + feats, names, uuids = self.gallery.get_centroids() + except Exception as e: + log.warning("auto-enroll: could not read gallery centroids: %s", e) + return None, "", 0.0 + if feats is None or feats.shape[0] == 0: + # Empty gallery — nothing to merge with + return None, "", 0.0 + + # Mean of buffer embeddings, L2-normalized + buf_mean = np.mean(np.stack(buf.embeddings, axis=0), axis=0) + n = float(np.linalg.norm(buf_mean)) + if n < 1e-12: + return None, "", 0.0 + buf_mean = (buf_mean / n).astype(np.float32) + + # Cosine sims against all gallery centroids + sims = feats @ buf_mean + # Sort descending so we can pick the best ANON match (not just best + # overall, which might be a named UUID we must NOT merge into). + order = np.argsort(-sims) + + for idx in order: + j = int(idx) + cand_sim = float(sims[j]) + cand_name = names[j] + cand_uuid = uuids[j] + if cand_sim < self.merge_thr: + # Beyond this point all candidates are below threshold — + # nothing left to consider. + break + if cand_name and cand_name.strip(): + # Named UUID — skip. Even if this is the closest match, we + # refuse to auto-merge into a named identity. Log so it's + # visible in demo what got rejected. + log.debug( + "auto-enroll merge_target SKIP named uuid=%s name=%r sim=%.3f " + "(named UUIDs never auto-merged into; will create new anon)", + cand_uuid, + cand_name, + cand_sim, + ) + continue + # Anonymous match — valid merge target + log.info( + "auto-enroll merge_target FOUND anon uuid=%s sim=%.3f " + "(sample fold-in)", + cand_uuid, + cand_sim, + ) + return cand_uuid, cand_name, cand_sim + + return None, "", 0.0 + + @staticmethod + def _min_pairwise_sim(embeddings: List[np.ndarray]) -> float: + """Return the WORST pairwise cosine sim across the buffer. + + Using the worst pair (not the average) is intentional — one bad + sample in a tight cluster shouldn't enroll, even if every other + pair is fine. The threshold acts as a sanity guard against + BoTSORT track-id mix-ups. + """ + if len(embeddings) < 2: + return 1.0 # vacuously tight + # Embeddings are already L2-normalized; dot product == cosine. + M = np.stack(embeddings, axis=0) + sims = M @ M.T + # Zero the diagonal so we don't pick up self-similarity + n = sims.shape[0] + sims[np.arange(n), np.arange(n)] = 1.0 + # We want the WORST off-diagonal element. + off_diag = sims[~np.eye(n, dtype=bool)] + return float(off_diag.min()) diff --git a/src/om1_vlm/anonymizationSys/face_recog_stream/config.py b/src/om1_vlm/anonymizationSys/face_recog_stream/config.py new file mode 100644 index 0000000..0c4ff44 --- /dev/null +++ b/src/om1_vlm/anonymizationSys/face_recog_stream/config.py @@ -0,0 +1,729 @@ +from __future__ import annotations + +from typing import Any, Dict + +# ============================================================================ +# 1. Face detection (SCRFD) +# ============================================================================ + +SCRFD_SIZE = 640 +# SCRFD input resolution (square). 640 standard; 480 faster but worse on +# small faces. +# - Used in: scrfd.py TRTSCRFD(size=...) +# - cfg key: none (engine compile-time) +# - Hot-tunable: NO + +DETECTION_CONFIDENCE = 0.5 +# Minimum face detection confidence. Below this, the detection is dropped +# before NMS. Lower → more recall (find more faces including profiles); higher +# → more precision (only frontal/clear faces). +# - Used in: scrfd.detect(conf=...), face_tracker BoTSORT init, gallery/add_raw +# - cfg key: "conf" +# - Hot-tunable: YES via /config +# - Example: at 0.5, you reliably catch frontal faces ~3m away. At 0.3 you +# also catch partial profiles but get more false-positive box wobbles. + +DETECTION_NMS = 0.4 +# NMS IoU threshold. Two boxes with IoU above this are deduped. +# Higher → more aggressive dedup (must overlap a lot to merge); lower → more +# aggressive merging (closer faces get merged into one box). +# - Used in: scrfd.nms_thresh (pushed live on /config update) +# - cfg key: "nms" +# - Hot-tunable: YES + +DETECTION_MAX_NUM = 0 +# Hard cap on faces returned per frame. 0 = no limit. Useful when crowds +# spike and you want to bound the work. +# - Used in: scrfd.detect(max_num=...) +# - cfg key: "max_num" +# - Hot-tunable: YES + + +# ============================================================================ +# 2. Recognition (AdaFace + tier-aware decision) +# ============================================================================ + +SIM_THR = 0.55 +# Cosine sim threshold below which we treat a match as 'unknown' regardless +# of margin. Top-1 floor. +# - Used in: selfie_logic.recognize_from_sims(min_sim=...), +# face_tracker.FaceTracker(sim_thr=...) +# - cfg key: "sim_thr" +# - Hot-tunable: YES +# - Calibration: collect ~50 (same-person, different-frame) pairs and ~500 +# (different-person) pairs. Plot cosine sim distributions. Pick sim_thr at +# the elbow of the cross-person distribution — typically 0.40-0.55 for +# AdaFace IR101. Higher → more rejections in bad lighting; lower → more +# false accepts. +# - Example: at 0.55, wendy in office light → match (sim=0.65). wendy in dim +# hallway → might fall to 0.50 and become 'unknown'. Lower to 0.45 if this +# happens too often. + +STRICT_MARGIN = 0.08 +# top1 - top2 margin required for the 'confident' recognition tier. When +# the top-1 candidate's similarity exceeds top-2's by at least this much, we +# trust the match without verification. +# - Used in: selfie_logic.recognize_from_sims(strict_margin=...), +# face_tracker.FaceTracker(strict_margin=...) +# - cfg key: "strict_margin" +# - Hot-tunable: YES +# - Calibration: with a small gallery (<50 ids), this can be tight (0.06-0.08). +# With a large gallery (>200 ids), near-collisions are more common, so +# loosen to 0.10-0.12 to avoid false confidence. +# - Example: top1=0.62 (wendy), top2=0.51 (alice) → margin=0.11 → confident. +# top1=0.62 (wendy), top2=0.58 (alice) → margin=0.04 → not confident. + +LOOSE_MARGIN = 0.04 +# top1 - top2 margin required for the 'tentative' recognition tier. Between +# strict_margin and loose_margin, we return the candidate but flag it as +# tentative — LLM should verify ('are you wendy?') instead of greeting by name. +# - Used in: same as strict_margin +# - cfg key: "loose_margin" +# - Hot-tunable: YES +# - Constraint: must be < strict_margin. +# - Example: top1=0.62, top2=0.57 → margin=0.05 → tentative. +# top1=0.62, top2=0.59 → margin=0.03 → uncertain (don't commit). + + +# ============================================================================ +# 3. Face tracking + voting (face_tracker.FaceTracker) +# ============================================================================ + +RECOG_INTERVAL_SEC = 0.2 +# Seconds between recognition attempts per track. Recognition is the +# expensive operation (AdaFace TRT inference); we don't need to re-recognize +# every frame for the same track. Lower → faster identification of new +# tracks; higher → lower GPU load. +# - Used in: face_tracker.FaceTracker(recog_interval=...) +# - Hot-tunable: NO (sticky) +# - Example: at 0.2s and 15fps, every 3rd frame we run recognition for a +# given track. A new track gets 3 votes in ~0.6s → identified. + +VOTE_FRAMES = 3 +# Number of recognition votes to collect before deciding identity. More +# votes = more robust against single-frame errors; fewer = faster decisions. +# - Used in: face_tracker.FaceTracker(vote_frames=...) + _finalize_identity +# - Hot-tunable: NO +# - Example: 3 votes with vote_threshold=0.5 means 2 of 3 must agree. + +VOTE_THRESHOLD = 0.5 +# Fraction of votes a single name needs to win. 0.5 = simple majority. +# - Used in: face_tracker.FaceTracker(vote_threshold=...) + _finalize_identity +# - Hot-tunable: NO +# - Example: vote_frames=3, vote_threshold=0.5 → 2/3 wins. +# vote_frames=3, vote_threshold=0.7 → all 3 must agree. + +MAX_RECOG_ATTEMPTS = 10 +# Hard cap on recognition attempts per track before forcing a decision. +# Prevents indefinite 'identifying...' state on tracks that never get a +# clear majority. +# - Used in: face_tracker.FaceTracker(max_recog_attempts=...) +# - Hot-tunable: NO + +TRACK_BUFFER = 30 +# BoTSORT frames to keep a 'lost' track alive (waiting for re-detection). +# Higher = better at handling brief occlusions but increases the chance of +# track-id mixups when two people swap positions. +# - Used in: face_tracker.FaceTracker(track_buffer=...) +# - Hot-tunable: NO +# - Frame-count, scales with FPS: 30 frames @ 15fps = 2 seconds of occlusion +# tolerance. If you change CAMERA_FPS, scale this accordingly to maintain +# the same wall-clock duration. +# - Example: at 15fps, a face occluded for <2s gets the same track_id back +# via BoTSORT's Kalman prediction. After 2s, occlusion timeout fires and +# a new track_id is assigned on re-detection. + +ARC_MAX_BS = 8 +# AdaFace TRT max batch size. Capped by the engine's optimization profile. +# - Used in: face_tracker.FaceTracker(arc_max_bs=...) +# - Hot-tunable: NO (engine compile-time) + +RE_IDENTIFY_INTERVAL_SEC = 1.0 +# Seconds before retrying recognition on tracks that came back 'unknown' +# or 'tentative'. Handles the 'walking closer to camera' case — face that +# was blurry at distance becomes clear, give it another shot. +# - Used in: face_tracker.FaceTracker(re_identify_interval=...) +# - Hot-tunable: NO +# - Example: an unknown track tries again every 1 second. AutoEnroller may +# have committed a UUID by then, in which case re-recognition succeeds. +# - Constraint check: must be < AUTO_ENROLL_STALE_TRACK_SEC (5.0) so the +# AutoEnroller buffer for a track in the unknown→re-identify gap doesn't +# get GC'd before observations resume. 1.0 < 5.0, safe. + + +# ============================================================================ +# 4. Auto-enrollment (auto_enroller.AutoEnroller) +# ============================================================================ + +AUTO_ENROLL_MIN_SAMPLES = 3 +# How many quality samples must accumulate before auto-enroll commits a +# new UUID. Lower → faster enrollment but more risk of misenrollment from +# BoTSORT track-id swaps. +# - Used in: auto_enroller.AutoEnroller(n_samples_required=...) +# - CLI: --auto-enroll-min-samples +# - Hot-tunable: NO (constructor-time) +# - Example: 3 with novelty_thr=0.97 and min_unknown_sec=1.0 means at least +# 3 distinct-enough samples within at least 1 second of observation. + +AUTO_ENROLL_MAX_BUFFER = 5 +# Maximum samples kept per track in the rolling buffer. When full, oldest +# is dropped. Should be ≥ n_samples_required. +# - Used in: auto_enroller.AutoEnroller(max_buffer=...) +# - CLI: --auto-enroll-max-buffer +# - Hot-tunable: NO + +AUTO_ENROLL_TIGHTNESS = 0.55 +# Required pairwise cosine similarity (worst pair) across the sample +# buffer before enrollment fires. Filters out tracks where BoTSORT confused +# two visitors during occlusion. +# - Used in: auto_enroller.AutoEnroller(min_pairwise_sim=...) +# - CLI: --auto-enroll-tightness +# - Hot-tunable: NO +# - Calibration: should be lower than SIM_THR (0.55) since within-person sim +# varies with lighting/angle. 0.55 is a conservative start; tune down to +# ~0.45 if too few auto-enrollments fire, up to 0.60 if you see polluted +# UUIDs (samples of two people enrolled together). + +AUTO_ENROLL_MIN_UNKNOWN_SEC = 1.0 +# Track must have been visible/unknown for this many seconds before +# auto-enroll considers it. Filters out walk-throughs that produce a few +# quick samples but aren't really "engaged with the robot". + +# Why 1.0s +# -------- +# At 1.5s the robot feels sluggish — a stranger walks up, stops, and waits +# 1.5+s before being enrolled. At 1.0s the enrollment happens just as the +# robot finishes saying "hi" — feels natural. + +# How this composes with other gates +# ---------------------------------- +# Auto-enroll commits when ALL of these are true: +# - sample_count ≥ n_required (3) +# - time elapsed ≥ min_unknown_sec +# - pairwise tightness ≥ min_pairwise_sim +# - each sample passes novelty + consistency checks + +# Other safeguards already filter walk-throughs: +# - min_face_pixels=30 rejects distant pedestrians +# - min_pairwise_sim=0.55 rejects samples where the face is moving/changing +# too fast (walk-throughs) +# - novelty_thr requires within-person variation (a single passing glance +# rarely produces 3 distinct embeddings) + +# So min_unknown_sec is one of multiple independent filters; it doesn't +# need to be conservative on its own. + +# - Used in: auto_enroller.AutoEnroller(min_unknown_sec=...) +# - CLI: --auto-enroll-min-unknown-sec +# - Hot-tunable: NO +# - Calibration: at demo time, watch enrollment latency. If too many +# walk-throughs enroll: raise to 1.5s. If robot feels slow: lower +# to 0.7-0.8s. + +AUTO_ENROLL_MIN_FACE_PX = 26 +# Minimum bbox short side for an auto-enroll sample. Below this, samples +# are dropped — too small to produce reliable embeddings. + +# Sized for 640×480 camera (Unitree G1 default). 30 px short side ≈ 6.25% +# of frame height, equivalent to a person ~2-3m from the camera at ~60° field of view. + +# At higher camera resolutions, scale up to maintain the same physical +# distance threshold: +# - 640×480 → 30 px +# - 1280×720 → 50 px +# - 1920×1080 → 75 px + +# - Used in: auto_enroller.AutoEnroller(min_face_pixels=...) +# - CLI: --auto-enroll-min-face-px +# - Hot-tunable: NO +# - Note: this is slightly looser than the per-frame recognition gate would +# be — auto-enroll wants multiple samples anyway, so a single marginal +# one isn't fatal. The pairwise-tightness check filters out noisy +# embeddings before commit. + + +AUTO_ENROLL_STALE_TRACK_SEC = 5.0 +# Drop a track's buffer if no observe() in this long. Saves memory on +# tracks that walked out of frame. +# - Used in: auto_enroller.AutoEnroller(stale_track_sec=...) +# - Hot-tunable: NO +# - Constraint: must be > RE_IDENTIFY_INTERVAL_SEC (1.0) so buffers +# survive the gap between finalize→unknown and re-identify resuming +# observations. 5.0 > 1.0, safe. + +# Dedicated auto-enroll quality gates (independent of the /selfie gates). +# A face must clear BOTH before auto-enroll will buffer/commit it, so only +# clear, frontal faces enter the gallery. These do NOT affect detection/ +# tracking/recognition (the global DETECTION_CONFIDENCE stays low so the +# robot still SEES profiles/far faces) — they only gate enrollment. +AUTO_ENROLL_MIN_CONF = 0.65 # min detector confidence to auto-enroll +AUTO_ENROLL_MIN_FRONTALITY = 0.6 # min frontality (0=off .. 1=dead-on) to auto-enroll + +AUTO_ENROLL_MERGE_THR = 0.50 +# Cosine sim threshold for merging an auto-enroll buffer into an existing +# UUID instead of creating a new one. Solves the 'known person in bad lighting' +# problem. + +# The problem +# ----------- +# SIM_THR=0.55 is the per-frame floor for recognition: below it, a face is +# labeled 'uncertain' even if it's a known person. So wendy in dim hallway +# (top1_sim ≈ 0.52 per frame) gets buffered by AutoEnroller. Without this +# threshold, AutoEnroller would create a SECOND UUID for wendy every time +# she walks into different lighting. + +# The fix +# ------- +# Before committing a new UUID, AutoEnroller computes the MEAN of all +# buffered embeddings (more reliable than any single frame — noise averages +# out, identity signal stays). If that mean matches any existing UUID's +# centroid with sim >= AUTO_ENROLL_MERGE_THR, append the samples to that +# UUID instead. + +# - Used in: auto_enroller.AutoEnroller(merge_thr=...) +# - CLI: --auto-enroll-merge-thr +# - Hot-tunable: NO +# - Calibration: should be LOWER than SIM_THR (so the merge path can rescue +# per-frame 'uncertain' decisions) but HIGHER than random-pair sim (~0.0-0.3 +# for cross-person on AdaFace) so true strangers still get new UUIDs. +# Setting to 0.0 disables the merge path (always creates new UUIDs). +# - Example: wendy buffered with mean sim 0.62 to her existing centroid → +# 0.62 >= 0.50 → merge. Stranger buffered with mean sim 0.18 to nearest +# centroid → 0.18 < 0.50 → new UUID. + + +# ============================================================================ +# 4b. Continuous sample refresh (sample_refresh.SampleRefreshManager) +# ============================================================================ +# +# Once a UUID exists in the gallery, every confident per-frame recognition +# is a potential opportunity to enrich its centroid with a new feature +# (different angle, lighting, expression). Three guards keep this safe: +# +# A. Match confidence — sim must exceed sim_thr + REFRESH_MIN_EXTRA_CONFIDENCE +# B. Per-UUID rate limit — one attempt per REFRESH_MIN_INTERVAL_SEC +# C. Diversity — new sample's sim to centroid in +# [REFRESH_MIN_DIVERSITY_SIM, REFRESH_MAX_DIVERSITY_SIM] +# +# The gallery itself enforces a hard cap of MAX_SAMPLES_PER_UUID = 50 with +# FIFO eviction, so disk stays bounded even with permissive guard settings. + +REFRESH_MIN_EXTRA_CONFIDENCE = 0.10 +# Guard A: minimum margin over sim_thr before a confident match qualifies +# for sample refresh. Effective threshold = sim_thr + this value. +# - Default: sim_thr=0.55, so refresh threshold = 0.65 +# - Higher → conservative (only enrich on rock-solid matches) +# - Lower → aggressive (more samples folded in, more risk of misattribution) +# - Used in: SampleRefreshManager(min_extra_confidence=...) +# - Hot-tunable: NO (would need to rebuild the manager) + +REFRESH_MIN_INTERVAL_SEC = 60.0 +# Guard B: minimum seconds between refresh attempts per UUID. Prevents a +# single session from flooding a UUID with near-identical samples. +# - Default: 60s. A 2-3 min conversation captures 2-3 samples, enough to +# pick up some pose/lighting variation without dominating disk I/O. +# - Combined with the 50-sample FIFO cap, the centroid stays diverse over +# long deployments. Bumping this to 600s (10 min) is also reasonable for +# very long-running deployments where intra-session diversity matters +# less than cross-session diversity. +# - Used in: SampleRefreshManager(min_refresh_interval_sec=...) + +REFRESH_MIN_DIVERSITY_SIM = 0.65 +# Guard C (lower bound): minimum sim between new sample's embedding and +# the UUID's centroid for refresh to accept. Below this, treat as +# likely-misidentification and skip. +# - Default: 0.65 (matches sim_thr + min_extra_confidence by design) +# - Higher → only very-similar samples get in (less drift risk) +# - Used in: gallery.refresh_sample(min_diversity_sim=...) + +REFRESH_MAX_DIVERSITY_SIM = 0.92 +# Guard C (upper bound): maximum sim between new sample's embedding and +# centroid. Above this, the sample is essentially a duplicate and adds no +# feature diversity — skip to keep ring-buffer slots for novel samples. +# - Default: 0.92 (allows minor pose/lighting variation, rejects near-clones) +# - Lower → only sufficiently-different angles get in (more diverse centroid) +# - Used in: gallery.refresh_sample(max_diversity_sim=...) + + +# ============================================================================ +# 4c. Gallery sample cap (gallery.UUIDGallery) +# ============================================================================ + +MAX_SAMPLES_PER_UUID = 50 +# Hard cap on samples stored per UUID. When add_sample / refresh_sample +# would push past this, the oldest sample is evicted (FIFO): +# JPG removed, embeddings.npy row 0 dropped, centroid_sum decremented. + +# Disk math: 50 samples × ~50KB per aligned 112x112 JPG = 2.5MB per UUID. +# 1000 UUIDs = 2.5GB. Effective ceiling for long-running deployments. + +# - Used in: UUIDGallery(max_samples_per_uuid=...) +# - Hot-tunable: NO (would need cleanup pass to shrink existing UUIDs) + +LAST_SEEN_DISK_PERSIST_INTERVAL_SEC = 60.0 +# How often (in seconds) ``touch_last_seen`` flushes ``last_seen_at`` to +# disk per UUID. In-memory ``rec.metadata["last_seen_at"]`` is updated on +# every call (cheap); disk write is rate-limited. + +# Without this rate limit, touch_last_seen would write metadata.json +# ~15×N times per second (N = visible faces), making it the dominant +# source of disk I/O. With 60s persist, the same load is ~1 write per +# minute per UUID — three orders of magnitude less churn. + +# The on-disk last_seen_at may lag the true last sighting by up to this +# many seconds across a crash, which is fine for the ``/gallery/sweep_stale`` +# use case (cleanup thresholds are typically days or months). + +# - Used in: UUIDGallery(last_seen_persist_interval_sec=...) +# - Hot-tunable: NO + + +# ============================================================================ +# 5. Selfie enrollment (http_api._do_selfie + selfie_logic) +# ============================================================================ + +SELFIE_WINDOW_SEC = 1.2 +# Maximum time the selfie loop waits for samples. After this, returns +# whatever it's collected (or error if < SELFIE_MIN_SAMPLES). +# - cfg key: "selfie_window_sec" +# - Hot-tunable: YES +# - Sized for 15fps camera: 1.2s × 15fps = 18 frames in the window. After +# quality gates (frontality, sharpness, brightness) filter ~40% out, +# we still net ~10 usable frames — well above SELFIE_MIN_SAMPLES=1 and +# enough room for SELFIE_MAX_SAMPLES=4. At 30fps you could drop to 0.8s. + +SELFIE_TAP_INTERVAL_SEC = 0.07 +# Sleep between selfie loop iterations to let new frames arrive. Should +# be slightly below the camera frame interval (1/fps) so we wake up just +# before each new frame is ready, without wasting CPU on busy-wait. +# - cfg key: "selfie_tap_interval_sec" +# - Hot-tunable: YES +# - For 15fps camera (frame interval = 66ms): 70ms is right. +# For 30fps (frame interval = 33ms): 30-35ms would be the equivalent. +# - Caveat: the selfie loop ALSO uses a frame_token check to detect fresh +# frames, so a slightly-too-short interval here doesn't double-process +# the same frame — just wastes a tiny bit of CPU. + +SELFIE_MIN_SAMPLES = 1 +# Adaptive sample count: minimum number of valid frames to enroll. min=1 +# lets static users enroll with just the first valid frame. +# - cfg key: "selfie_min_samples" +# - Hot-tunable: YES + +SELFIE_MAX_SAMPLES = 4 +# Maximum samples per selfie. Caps storage/embedding work per call. +# - cfg key: "selfie_max_samples" +# - Hot-tunable: YES + +SELFIE_MIN_ENGAGEMENT = 0.0025 +# Engagement score floor below which a face isn't considered a selfie +# target. score = (bbox_area / frame_area) × frontality, in [0, 1]. + +# Calibrated for 640×480 camera + SELFIE_MIN_FACE_PX=30 +# --------------------------------------------------------- +# The minimum sensible engagement is the score of a barely-passing face: +# 30 px frontal face → score = (30×30) / (640×480) × 1.0 = 0.00293. + +# 0.0025 sits just below this with a small margin to absorb detection +# box wobble (1-2 px jitter can shrink bbox area by ~10%). + +# Behavior at this threshold: +# - 30 px frontal face → 0.00293 ✓ pass +# - 30 px ¾-profile (f=0.85) → 0.00249 ✗ borderline +# - 25 px frontal face → 0.00203 ✗ too small +# - 40 px slight side (f=0.5) → 0.00260 ✓ pass +# - 70 px frontal at 1m → 0.0159 ✓ pass (typical close selfie) + +# If you change CAMERA_WIDTH/HEIGHT or SELFIE_MIN_FACE_PX, recompute: +# min_engagement = (min_face_px)² / (W × H) × 0.85 +# where the 0.85 factor leaves margin for box jitter + frontality < 1. + +# - cfg key: "selfie_min_engagement" +# - Hot-tunable: YES +# - Multi-face role: engagement score is also used to PICK the best face +# when multiple are detected (argmax). The floor here additionally +# enables the ambiguity check (skips frames where top-1 and top-2 are +# both above floor and close in score). + +SELFIE_AMBIGUITY_RATIO = 0.80 +# top2/top1 engagement ratio above which we treat the frame as 'ambiguous' +# (two people equally addressing the robot). Frame skipped. +# - cfg key: "selfie_ambiguity_ratio" +# - Hot-tunable: YES +# - Example: 0.80 means if two faces have engagement scores 0.05 and 0.045, +# ratio = 0.9 > 0.80 → ambiguous, skip frame. + +SELFIE_MIN_FACE_PX = 30 +# Selfie per-frame quality gate: bbox short side in pixels. + +# Sized for 640×480 camera (Unitree G1 default). 30 px short side ≈ 6.25% +# of frame height, equivalent to a person ~2-3m from the camera at ~60° field of view. + +# At higher camera resolutions (e.g. 1280×720) you can raise this to keep +# the same physical distance threshold: +# - 640×480 → 30 px +# - 1280×720 → 50 px +# - 1920×1080 → 75 px + +# - cfg key: "selfie_min_face_px" +# - Hot-tunable: YES + +SELFIE_MIN_CONF = 0.65 +# Selfie per-frame quality gate: detector confidence. +# - cfg key: "selfie_min_conf" +# - Hot-tunable: YES + +SELFIE_MIN_FRONTALITY = 0.4 +# Selfie per-frame quality gate: frontality from 5-point landmarks, [0,1]. +# - cfg key: "selfie_min_frontality" +# - Hot-tunable: YES + +SELFIE_SHARP_THR = 5.0 +# Selfie per-frame quality gate: Laplacian variance threshold for blur +# detection. Low value = blurry image → reject. +# - cfg key: "selfie_sharp_thr" +# - Hot-tunable: YES + +SELFIE_BRIGHTNESS_MIN = 30 +# Selfie per-frame quality gate: minimum grayscale mean. +# - cfg key: "selfie_brightness_min" +# - Hot-tunable: YES + +SELFIE_BRIGHTNESS_MAX = 230 +# Selfie per-frame quality gate: maximum grayscale mean. +# - cfg key: "selfie_brightness_max" +# - Hot-tunable: YES + +SELFIE_NOVELTY_THR = 0.93 +# Cosine sim threshold above which a new selfie frame is 'too similar to +# what we already have' → skip (novelty check). +# - cfg key: "selfie_novelty_thr" +# - Hot-tunable: YES +# - Example: 0.93 means new frame must have at least ~7% angular variation +# from the running mean. + +SELFIE_CONSISTENCY_THR = 0.50 +# Cosine sim threshold below which a selfie frame is 'a different person +# from the one we started with' → skip (consistency check). +# - cfg key: "selfie_consistency_thr" +# - Hot-tunable: YES + +SELFIE_MERGE_THR = 0.5 +# Cosine sim threshold for merging a selfie into an existing same-name UUID. +# - Used in: selfie_logic.resolve_target_id +# - cfg key: "selfie_merge_thr" +# - Hot-tunable: YES +# - Example: 0.45 means if a new wendy selfie matches an existing wendy UUID +# at sim >= 0.45, append samples instead of creating a new UUID. + +SELFIE_CROSS_NAME_THR = 0.60 +# Cosine sim threshold above which a cross-name match triggers rejection +# (without force=True). Prevents 'Bob accidentally enrolled as Wendy'. +# - Used in: selfie_logic.resolve_target_id +# - cfg key: "selfie_cross_name_thr" +# - Hot-tunable: YES +# - Example: 0.60 means if face matches an existing UUID at sim >= 0.60 but +# the names differ → reject and return 'face_belongs_to'. + + +# ============================================================================ +# 6. Pixelation / privacy blur +# ============================================================================ + +BLUR_ENABLED = False +# Whether to apply privacy pixelation at all. +# - cfg key: "blur" +# - Hot-tunable: YES + +BLUR_MODE = "all" +# Which faces to blur. Options: "all", "known", "unknown". +# - cfg key: "blur_mode" +# - Hot-tunable: YES +# - Example: "unknown" blurs only unrecognized faces (useful when you want +# to show only enrolled people clearly). + +PIXEL_BLOCKS = 8 +# Pixelation granularity: number of pixel blocks across the short side of +# the bbox. Smaller = coarser pixelation; larger = finer. +# - cfg key: "pixel_blocks" +# - Hot-tunable: YES + +PIXEL_MARGIN = 0.25 +# Bbox expansion (fraction of size) before pixelation. Helps hide hair/ears +# that sit outside the detection box. +# - cfg key: "pixel_margin" +# - Hot-tunable: YES + +PIXEL_NOISE = 0.0 +# Gaussian noise sigma added to pixelated area. 0 = off. Helps prevent +# pixel-perfect un-pixelation attacks. +# - cfg key: "pixel_noise" +# - Hot-tunable: YES + + +# ============================================================================ +# 7. Pose detection + fall detection +# ============================================================================ + +POSE_ENABLED = False +# Whether to run pose detection (YOLO11s-pose). Disabled by default for +# cost. Enable with --pose. + +POSE_CONF = 0.5 +# YOLO pose detector confidence threshold. +# - cfg key: "pose_conf" +# - Hot-tunable: YES + +POSE_MAX_NUM = 10 +# Maximum persons per frame for pose detection. +# - cfg key: "pose_max_num" +# - Hot-tunable: YES + +FALL_DETECTION_ENABLED = False +# Whether to run fall detection. Requires POSE_ENABLED. +# - cfg key: "fall_detection" +# - Hot-tunable: YES + +# Drawing options — all hot-tunable via /config +DRAW_BOXES = False +DRAW_NAMES = False +DRAW_SKELETON = False +DRAW_POSE_BOXES = False +DRAW_FALL_STATUS = False +SHOW_FPS = False + + +# ============================================================================ +# 8. Recognition strategy (main loop crowd handling) +# ============================================================================ + +RECOG_TOPK = 4 +# Maximum faces per frame to send to AdaFace. Caps per-frame GPU work in +# crowds. Should be ≤ AdaFace engine's max batch size. +# - CLI: --recog-topk +# - Hot-tunable: NO + +CROWD_THR = 12 +# Hard crowd cap: if detected faces exceed this, skip recognition entirely +# for this frame (still draw boxes and blur). Prevents GPU starvation when +# a crowd shows up. +# - cfg key: "crowd_thr" +# - Hot-tunable: YES + + +# ============================================================================ +# 9. Camera / RTSP / HTTP +# ============================================================================ + +CAMERA_DEVICE = "/dev/video0" +CAMERA_WIDTH = 640 +CAMERA_HEIGHT = 480 +# Camera resolution. Unitree G1's onboard camera default is 640×480. + +# All bbox-size thresholds (SELFIE_MIN_FACE_PX=30, AUTO_ENROLL_MIN_FACE_PX=30) +# are sized for this resolution. If you bump to a higher resolution camera, +# scale those thresholds proportionally: +# 640×480 → 30 px (current) +# 1280×720 → 50 px +# 1920×1080 → 75 px +# to keep the same effective "person distance" threshold. +CAMERA_FPS = 15 +# Capture frame rate. The Unitree G1 onboard camera runs at 15 fps. + +# All wall-clock-based parameters (recog_interval, min_unknown_sec, +# re_identify_interval, stale_track_sec) are independent of this — they +# measure seconds, not frames. + +# Frame-count parameters that DO scale with fps: +# - TRACK_BUFFER (currently 30 = 2s @ 15fps) +# - gc_stale modulo in run.py main loop (15 = ~1s @ 15fps) + +LOCAL_RTSP_URL = "rtsp://localhost:8554/top_camera" +RAW_LOCAL_RTSP_URL = "rtsp://localhost:8554/top_camera_raw" + +HTTP_HOST = "127.0.0.1" +HTTP_PORT = 6793 +HTTP_LOOKBACK_SEC = 10.0 +# Default lookback window for /who queries (seconds). + + +# ============================================================================ +# DEFAULTS dict — seeds the runtime cfg dict in run.py +# ============================================================================ +# +# Only keys present here are hot-tunable via /config. Adding a new key here +# (plus a setter on the relevant component) makes a param /config-tunable. +# Removing an entry locks that param to its startup value. + +DEFAULTS: Dict[str, Any] = { + # Detection + "conf": DETECTION_CONFIDENCE, + "nms": DETECTION_NMS, + "max_num": DETECTION_MAX_NUM, + # Recognition tier + "sim_thr": SIM_THR, + "strict_margin": STRICT_MARGIN, + "loose_margin": LOOSE_MARGIN, + # Blur + "blur": BLUR_ENABLED, + "blur_mode": BLUR_MODE, + "pixel_blocks": PIXEL_BLOCKS, + "pixel_margin": PIXEL_MARGIN, + "pixel_noise": PIXEL_NOISE, + # Pose + "pose": POSE_ENABLED, + "pose_conf": POSE_CONF, + "pose_max_num": POSE_MAX_NUM, + "fall_detection": FALL_DETECTION_ENABLED, + # Crowd + "crowd_thr": CROWD_THR, + # Selfie pipeline (all hot-tunable via /config) + "selfie_window_sec": SELFIE_WINDOW_SEC, + "selfie_tap_interval_sec": SELFIE_TAP_INTERVAL_SEC, + "selfie_min_samples": SELFIE_MIN_SAMPLES, + "selfie_max_samples": SELFIE_MAX_SAMPLES, + "selfie_min_engagement": SELFIE_MIN_ENGAGEMENT, + "selfie_ambiguity_ratio": SELFIE_AMBIGUITY_RATIO, + "selfie_min_face_px": SELFIE_MIN_FACE_PX, + "selfie_min_conf": SELFIE_MIN_CONF, + "selfie_min_frontality": SELFIE_MIN_FRONTALITY, + "selfie_sharp_thr": SELFIE_SHARP_THR, + "selfie_brightness_min": SELFIE_BRIGHTNESS_MIN, + "selfie_brightness_max": SELFIE_BRIGHTNESS_MAX, + "selfie_novelty_thr": SELFIE_NOVELTY_THR, + "selfie_consistency_thr": SELFIE_CONSISTENCY_THR, + "selfie_merge_thr": SELFIE_MERGE_THR, + "selfie_cross_name_thr": SELFIE_CROSS_NAME_THR, + # Drawing + "show_fps": SHOW_FPS, + "draw_skeleton": DRAW_SKELETON, + "draw_pose_boxes": DRAW_POSE_BOXES, + "draw_fall_status": DRAW_FALL_STATUS, +} + + +# ============================================================================ +# Hot-tunability quick reference +# ============================================================================ +# +# YES — via POST /config {"set": {...}} while system is running: +# conf, nms, max_num +# sim_thr, strict_margin, loose_margin +# blur, blur_mode, pixel_blocks, pixel_margin, pixel_noise +# pose, pose_conf, pose_max_num, fall_detection +# crowd_thr +# selfie_* (all selfie pipeline knobs) +# show_fps, draw_* +# +# NO — restart required (controllable via CLI flags at startup): +# AUTO_ENROLL_* (auto_enroller constructor-time) +# RECOG_INTERVAL_SEC, VOTE_FRAMES, VOTE_THRESHOLD, MAX_RECOG_ATTEMPTS, +# TRACK_BUFFER, ARC_MAX_BS, RE_IDENTIFY_INTERVAL_SEC +# (face_tracker constructor-time) +# SCRFD_SIZE, AdaFace engine path (engine compile-time) +# Camera / RTSP / HTTP bindings +# +# To make a constructor-time param hot-tunable: +# 1. Add the key to DEFAULTS above +# 2. Add a setter on the relevant component (like FaceTracker.set_thresholds) +# 3. Push the value from the main loop's per-frame cfg snapshot diff --git a/src/om1_vlm/anonymizationSys/face_recog_stream/face_tracker.py b/src/om1_vlm/anonymizationSys/face_recog_stream/face_tracker.py index e1a6617..b2c5685 100644 --- a/src/om1_vlm/anonymizationSys/face_recog_stream/face_tracker.py +++ b/src/om1_vlm/anonymizationSys/face_recog_stream/face_tracker.py @@ -1,17 +1,31 @@ """ -Face tracker with BoTSORT + low-frequency face recognition. +Face tracker with BoTSORT + low-frequency, tier-aware face recognition. Architecture: Every frame: SCRFD detect → BoTSORT track (persistent IDs) - Low freq: For unidentified tracks → AdaFace embed → vote → assign identity - Once matched: Identity sticks to track ID until track is lost + Low freq: For unidentified tracks → AdaFace embed → margin-aware + recognition → multi-frame voting → assign identity + Once matched: Identity sticks to track ID until track is lost. + +Recognition tiers (see selfie_logic.recognize_from_sims): + - "confident": top1 - top2 margin >= strict_margin → trust this match + - "tentative": top1 - top2 margin >= loose_margin → likely but ask to verify + - "uncertain": below loose_margin or top1 below sim_thr → "unknown" vote + +Voting decision (see _finalize_identity): + 1. If any name has >= vote_threshold fraction of CONFIDENT votes → identified + with tier="confident". + 2. Else if any name has >= vote_threshold fraction of CONFIDENT+TENTATIVE + votes → identified with tier="tentative" (caller should verify with user). + 3. Otherwise → unknown. Usage: - tracker = FaceTracker(arc=arc_engine) + tracker = FaceTracker(arc=arc_engine, sim_thr=0.45, + strict_margin=0.08, loose_margin=0.04) tracker.set_gallery(feats, labels) # Per frame: results = tracker.update(frame, dets, kpss) - # results = [TrackResult(track_id=1, bbox=..., name="wendy (0.72)", ...), ...] + # results = [TrackResult(track_id=1, raw_name="wendy", tier="confident", ...), ...] """ from __future__ import annotations @@ -20,49 +34,149 @@ import time from collections import Counter from dataclasses import dataclass, field -from typing import Dict, List, Optional, Tuple +from typing import Callable, Dict, List, Optional, Tuple import cv2 import numpy as np +from . import selfie_logic as sl + log = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Public dataclasses +# --------------------------------------------------------------------------- + + @dataclass class TrackResult: - """Per-track output each frame.""" + """Per-track output each frame. + + Fields + ------ + track_id : int + Persistent BoTSORT id; survives short occlusions. + bbox : (x1, y1, x2, y2) + Face bounding box in image coordinates. + conf : float + BoTSORT detection confidence for this frame. + raw_name : str | None + Plain identity label, e.g. ``"wendy"``, or ``None`` if not yet voted. + Use this for downstream tracking (WhoTracker, fall match, etc.). + name : str | None + Display string with score, e.g. ``"wendy (0.72)"`` or ``"wendy? (0.62)"``. + Use this for on-frame overlays/drawing. + sim : float + Average cosine similarity over the winning votes (0 if unknown). + tier : str + ``"confident"`` / ``"tentative"`` / ``"uncertain"``. Drives downstream + UX (e.g. whether the robot greets by name or asks to verify). + uuid : str | None + Stable identity ID from the gallery (UUID4 hex), or ``None`` if the + track is unidentified. Names can change via ``set_name``; UUIDs + cannot. Downstream layers (HttpAPI, audit log) should reference + identities by UUID rather than name. + is_known : bool + True only when ``tier == "confident"``. Kept for back-compat with + callers (e.g. blur logic) that previously had only a boolean signal. + """ track_id: int - bbox: Tuple[int, int, int, int] # x1, y1, x2, y2 + bbox: Tuple[int, int, int, int] conf: float - name: Optional[str] = None # display name (e.g. "wendy (0.72)" or "wendy? (0.62)") - sim: float = 0.0 # best similarity score - is_known: bool = False # True only after vote confirmed + raw_name: Optional[str] = None + name: Optional[str] = None + sim: float = 0.0 + tier: str = sl.TIER_UNCERTAIN + uuid: Optional[str] = None + is_known: bool = False + + +# --------------------------------------------------------------------------- +# Internal state +# --------------------------------------------------------------------------- @dataclass class _TrackIdentity: - """Internal identity state for one track.""" + """Internal identity state accumulated for one BoTSORT track. + + Lifecycle: new → identifying → (identified | unknown) + On re_identify_interval, "unknown" or low-confidence "identified" tracks + are reset back to "new" so recognition can retry on a clearer frame. + """ track_id: int - status: str = "new" # new → identifying → identified / unknown + status: str = "new" # new | identifying | identified | unknown + + # Decided identity (after _finalize_identity) name: Optional[str] = None + uuid: Optional[str] = None # gallery UUID of the winning name (None if unknown) sim: float = 0.0 - # Embedding votes collected during identification + tier: str = sl.TIER_UNCERTAIN + + # Votes collected during the identifying phase. + # Length-aligned lists: vote_names[i] is the predicted name for vote i, + # vote_uuids[i] is the gallery UUID that name maps to (parallel; needed + # because multiple UUIDs can share a name), vote_tiers[i] is the tier + # of that prediction, vote_sims[i] is the top-1 sim. "unknown" entries + # are kept (count toward total votes) but excluded from naming decisions + # in _finalize_identity. vote_names: List[str] = field(default_factory=list) + vote_uuids: List[Optional[str]] = field(default_factory=list) + vote_tiers: List[str] = field(default_factory=list) vote_sims: List[float] = field(default_factory=list) embeddings: List[np.ndarray] = field(default_factory=list) - # Timing + + # Timing & bookkeeping first_seen: float = 0.0 last_recog_time: float = 0.0 frames_seen: int = 0 recog_attempts: int = 0 - # Unknown capture state - unknown_since: float = 0.0 # when status first became unknown (0 = not unknown) + # When >0, marks when this track first became (or stayed) unknown. + # Used by the auto-enroll system to time how long a face has been + # unknown. + unknown_since: float = 0.0 + # ISO timestamp of the gallery's ``last_seen_at`` at the moment this + # track first identified to a known UUID. Captured BEFORE any + # touch_last_seen call from this session, so it reflects the + # PREVIOUS sighting of this UUID — not "right now". + # Stays sticky for the lifetime of the track session: even as the + # gallery's last_seen_at is updated every frame, this field doesn't + # change. That gives the LLM consistent "we've met before" / "newcomer" + # signal throughout a single conversation. + # None until the first confident identification. + session_last_seen_iso: Optional[str] = None + + # The last UUID this track was CONFIDENTLY identified as. Persists across + # re-identify resets (unlike `uuid`, which gets cleared when a track goes + # unknown), so we can detect when one continuous track flips from one + # confident identity to a different one — the trigger for auto-merging + # fragmented identities (e.g. a frontal vs a profile enrollment). + last_confident_uuid: Optional[str] = None + + +# --------------------------------------------------------------------------- +# FaceTracker +# --------------------------------------------------------------------------- class FaceTracker: - """BoTSORT face tracker with low-frequency recognition and multi-frame voting. + """BoTSORT face tracker with tier-aware recognition and multi-frame voting. + + Recognition flow per frame: + + 1. BoTSORT assigns a track_id to each SCRFD detection (persistent IDs). + 2. For tracks in ``"new"`` or ``"identifying"`` state that haven't been + re-checked within ``recog_interval`` seconds, queue them for AdaFace + embedding extraction. + 3. Embed in a single batched pass (one TRT call for all queued tracks). + 4. For each track, run :func:`selfie_logic.recognize_from_sims` to get + ``(name, tier, top1_sim, top2_sim)``. Push that into the track's + vote history. + 5. Once a track has ``vote_frames`` votes (or hit ``max_recog_attempts``), + call :meth:`_finalize_identity` to decide. Parameters ---------- @@ -71,17 +185,27 @@ class FaceTracker: recog_interval : float Seconds between recognition attempts per track (default 0.3). vote_frames : int - Recognition frames to collect before voting (default 3). + Number of votes to collect before deciding identity (default 3). vote_threshold : float - Fraction of votes needed to confirm identity (default 0.5). + Fraction of votes a single name needs to win (default 0.5). max_recog_attempts : int - Max attempts before marking as "unknown" (default 10). + Hard cap on recognition attempts per track (default 10). sim_thr : float - Cosine similarity threshold for match (default 0.4). + Top-1 cosine threshold below which a vote is "uncertain" (default 0.4). + strict_margin : float + ``top1 - top2`` required for a vote to be "confident" (default 0.08). + loose_margin : float + ``top1 - top2`` required for a vote to be at least "tentative" + (default 0.04). Must be < ``strict_margin``. track_buffer : int BoTSORT frames to keep lost tracks alive (default 60). arc_max_bs : int - Max batch size for AdaFace inference (default 8). + AdaFace TRT max batch size (default 8). + det_conf : float + SCRFD detection confidence; passed to BoTSORT for tier alignment. + re_identify_interval : float + Seconds before retrying recognition on "unknown" or marginal tracks. + 0 disables retry (default 3.0). """ def __init__( @@ -93,10 +217,18 @@ def __init__( vote_threshold: float = 0.5, max_recog_attempts: int = 10, sim_thr: float = 0.4, + strict_margin: float = 0.08, + loose_margin: float = 0.04, track_buffer: int = 60, arc_max_bs: int = 8, det_conf: float = 0.5, re_identify_interval: float = 3.0, + on_unknown_sample: Optional[ + "Callable[[int, np.ndarray, np.ndarray, Tuple[int,int,int,int], str, Optional[np.ndarray], float], Optional[str]]" + ] = None, + on_confident_sample: Optional[ + "Callable[[int, str, np.ndarray, np.ndarray, float], None]" + ] = None, ): self.arc = arc self.recog_interval = float(recog_interval) @@ -104,12 +236,31 @@ def __init__( self.vote_threshold = float(vote_threshold) self.max_recog_attempts = int(max_recog_attempts) self.sim_thr = float(sim_thr) + self.strict_margin = float(strict_margin) + self.loose_margin = float(loose_margin) self.arc_max_bs = int(arc_max_bs) self.re_identify_interval = float(re_identify_interval) - - # Gallery (set via set_gallery) + # Callback fired once per recognition pass per track, regardless of + # tier. Signature: (track_id, aligned_crop, embedding, bbox, tier). + # Used by AutoEnroller to buffer unknown faces and drop committed + # ones — keeping face_tracker unaware of storage concerns. + self.on_unknown_sample = on_unknown_sample + # Callback fired only on confident per-frame identifications. + # Signature: (track_id, uuid, aligned_crop, embedding, sim). + # Used by SampleRefreshManager for continuous-learning sample + # enrichment. Guard A (sim threshold) is the caller's responsibility. + self.on_confident_sample = on_confident_sample + # Fired when one continuous track's CONFIDENT identity changes from one + # UUID to a different one. Signature: (track_id, old_uuid, new_uuid). + # Set as a plain attribute (like on_unknown_sample) by run.py, wired to + # the auto-merge handler. None = feature off. + self.on_identity_flip: Optional[Callable[[int, str, str], None]] = None + + # Gallery (set via set_gallery). All three arrays are parallel — + # row i of _gal_feats has name _gal_labels[i] and UUID _gal_uuids[i]. self._gal_feats: Optional[np.ndarray] = None self._gal_labels: List[str] = [] + self._gal_uuids: List[str] = [] # BoTSORT tracker — align thresholds with SCRFD det_conf self._det_conf = float(det_conf) @@ -125,9 +276,13 @@ def __init__( # Current frame faces (for status queries) self._current_faces: list = [] + # ------------------------------------------------------------------ + # Setup / runtime config + # ------------------------------------------------------------------ + @staticmethod def _init_tracker(track_buffer: int, det_conf: float = 0.5): - """Initialize BoTSORT tracker with thresholds aligned to detection confidence.""" + """Initialize BoTSORT with thresholds aligned to detection confidence.""" from boxmot import BotSort high_thresh = max(det_conf, 0.3) @@ -154,15 +309,142 @@ def _init_tracker(track_buffer: int, det_conf: float = 0.5): with_reid=False, ) - def set_gallery(self, feats: Optional[np.ndarray], labels: List[str]) -> None: - """Update gallery embeddings and labels.""" + def set_gallery( + self, + feats: Optional[np.ndarray], + labels: List[str], + uuids: Optional[List[str]] = None, + ) -> None: + """Update gallery centroids, names, and UUIDs (called after enrollment). + + ``uuids`` is optional for back-compat — if omitted, an empty string is + used as a placeholder per row. Pass real UUIDs when using + ``UUIDGallery.get_centroids()`` so downstream layers can reference + identities by stable ID. + + Two side-effects on cached track identities (``self._identities``): + + 1. **Stale invalidation** — if a track's cached UUID no longer exists + in the new gallery (it got deleted), reset that track to "new" + status so it re-enters recognition on the next frame. Without this, + a track identified as "sean" before sean got deleted would keep + displaying "sean" until the BoTSORT track itself died. + + 2. **Name propagation** — if a track's cached UUID still exists but + its name was changed (e.g. anon_xxx → sean via /set_name), update + the cached ``ident.name`` in place. Without this, the displayed + label would lag the rename by up to ``re_identify_interval`` seconds + (the next recognition cycle), confusing demo viewers who expect + rename to update the video immediately. + """ self._gal_feats = feats self._gal_labels = list(labels) + if uuids is None: + self._gal_uuids = [""] * len(labels) + else: + if len(uuids) != len(labels): + raise ValueError( + f"uuids ({len(uuids)}) and labels ({len(labels)}) must be same length" + ) + self._gal_uuids = list(uuids) + + # Build uuid → current_name map for O(1) lookups + valid_uuids: Dict[str, str] = {} + for u, n in zip(self._gal_uuids, self._gal_labels): + if u: + valid_uuids[u] = n + + stale_count = 0 + renamed_count = 0 + for tid, ident in self._identities.items(): + # A confidently-remembered UUID that no longer exists (e.g. it was + # the loser of an auto-merge) must be forgotten, else it could + # spuriously re-trigger a flip. Do this regardless of current uuid. + if ( + ident.last_confident_uuid + and ident.last_confident_uuid not in valid_uuids + ): + ident.last_confident_uuid = None + if not ident.uuid: + continue + if ident.uuid not in valid_uuids: + # (1) Stale: the UUID this track was identified to is gone. + log.info( + "Track %d: identity %r (uuid=%s) removed from gallery — resetting", + tid, + ident.name, + ident.uuid, + ) + ident.status = "new" + ident.name = None + ident.uuid = None + ident.sim = 0.0 + ident.tier = sl.TIER_UNCERTAIN + ident.vote_names.clear() + ident.vote_uuids.clear() + ident.vote_tiers.clear() + ident.vote_sims.clear() + ident.embeddings.clear() + ident.recog_attempts = 0 + # Don't reset first_seen / frames_seen / unknown_since — + # those describe the track itself, not the identity. + stale_count += 1 + else: + # (2) UUID still valid — check if name changed (rename via + # /set_name, or claim via /selfie merging anon into named). + new_name = valid_uuids[ident.uuid] + # ident.name might be None for tracks that were anon (no name + # finalized yet); coerce both sides to plain strings for + # comparison. "" (anon) → "" (still anon) is a no-op; + # "" → "sean" is the rename / claim case. + cached_name = ident.name or "" + if cached_name != new_name: + log.info( + "Track %d: uuid=%s name updated %r → %r (live propagation)", + tid, + ident.uuid, + cached_name, + new_name, + ) + # Update cached name AND any votes for this UUID so the + # display rebuild picks up the change immediately. Note: + # we do NOT touch sim / tier / status — only the name. + ident.name = new_name if new_name else None + for i, vu in enumerate(ident.vote_uuids): + if vu == ident.uuid: + ident.vote_names[i] = new_name + renamed_count += 1 + if stale_count > 0 or renamed_count > 0: + log.info( + "set_gallery: reset=%d renamed=%d (%d valid UUIDs)", + stale_count, + renamed_count, + len(valid_uuids), + ) + + def set_thresholds( + self, + *, + sim_thr: Optional[float] = None, + strict_margin: Optional[float] = None, + loose_margin: Optional[float] = None, + ) -> None: + """Hot-update recognition thresholds. ``None`` means "leave as-is".""" + if sim_thr is not None: + self.sim_thr = float(sim_thr) + if strict_margin is not None: + self.strict_margin = float(strict_margin) + if loose_margin is not None: + self.loose_margin = float(loose_margin) def set_sim_thr(self, thr: float) -> None: - """Update similarity threshold at runtime.""" + """Back-compat shim — prefer ``set_thresholds(sim_thr=thr)``.""" self.sim_thr = float(thr) + # ------------------------------------------------------------------ + # Main per-frame entry point + # ------------------------------------------------------------------ + def update( self, frame: np.ndarray, @@ -178,12 +460,12 @@ def update( dets : ndarray (N, 5) SCRFD detections [x1, y1, x2, y2, conf]. kpss : ndarray (N, 5, 2) or None - 5-point landmarks per detection. + 5-point landmarks per detection (used for alignment). Returns ------- list[TrackResult] - One entry per tracked face. + One entry per active track this frame. """ now = time.time() H, W = frame.shape[:2] @@ -191,13 +473,13 @@ def update( # Run BoTSORT tracks = self._run_tracker(dets, frame) - # Match tracks to SCRFD detections (for landmarks) + # Match tracks back to SCRFD detections (for landmarks) track_det_map = self._match_tracks_to_dets(tracks, dets, kpss) - # Build results + collect tracks needing recognition + # Build results + collect tracks that still need recognition self._active_ids = set() results: List[TrackResult] = [] - need_recog: List[Tuple[int, np.ndarray, Optional[np.ndarray]]] = [] + need_recog: List[Tuple[int, np.ndarray, Optional[np.ndarray], float]] = [] for track in tracks: x1, y1, x2, y2 = map(int, track[:4]) @@ -220,130 +502,290 @@ def update( ident = self._identities[track_id] ident.frames_seen += 1 - # Track how long this face has been unknown - if ident.status == "identified": - ident.unknown_since = 0.0 # reset when identified + # Track how long this face has been unknown (for auto-enroll later) + if ident.status == "identified" and ident.tier == sl.TIER_CONFIDENT: + ident.unknown_since = 0.0 elif ident.unknown_since == 0.0: - ident.unknown_since = now # start timing - - # Re-identify: if "unknown" or low-confidence "identified" and enough - # time has passed, reset voting state so recognition retries. - # Handles the "walking closer" scenario where face gets clearer over time. - if ( - self.re_identify_interval > 0 - and (now - ident.last_recog_time) >= self.re_identify_interval - ): - if ident.status == "unknown": - log.debug( - "Track %d: re-identify (was unknown, %.1fs since last attempt)", - track_id, - now - ident.last_recog_time, - ) - ident.status = "new" - ident.vote_names.clear() - ident.vote_sims.clear() - ident.embeddings.clear() - ident.recog_attempts = 0 - elif ident.status == "identified" and ident.sim < self.sim_thr + 0.1: - # Low-confidence match — re-verify when face is clearer - log.debug( - "Track %d: re-verify '%s' (sim=%.3f, marginal)", - track_id, - ident.name, - ident.sim, - ) - ident.status = "new" - ident.vote_names.clear() - ident.vote_sims.clear() - ident.embeddings.clear() - ident.recog_attempts = 0 + ident.unknown_since = now - # Should we run recognition for this track? + # Re-identify policy: if state is stale, reset votes so recognition + # retries on what is hopefully a clearer frame. + self._maybe_reset_for_reidentify(ident, now) + + # Decide whether this track needs another recognition attempt. + # NOTE: we intentionally do NOT require a non-empty gallery here. + # Even with zero identities, we still want to embed the face and + # forward it to AutoEnroller (the empty-gallery branch below sets + # tier=uncertain and calls _fire_unknown_callback). Excluding empty + # gallery here would silently break auto-enrollment on a fresh + # install. needs_recog = ( ident.status in ("new", "identifying") - and self._gal_feats is not None - and self._gal_feats.size > 0 and (now - ident.last_recog_time) >= self.recog_interval and ident.recog_attempts < self.max_recog_attempts ) if needs_recog: det_kps = track_det_map.get(track_id, {}).get("kps") - need_recog.append((track_id, np.array([x1, y1, x2, y2]), det_kps)) + need_recog.append((track_id, np.array([x1, y1, x2, y2]), det_kps, conf)) - # Build display name based on current status - name_display, is_known, sim = self._get_display_name(ident) + # Build display + raw name based on current status/tier + raw_name, display_name, is_known, sim, tier, uuid = self._build_display( + ident + ) results.append( TrackResult( track_id=track_id, bbox=(x1, y1, x2, y2), conf=conf, - name=name_display, + raw_name=raw_name, + name=display_name, sim=sim, + tier=tier, + uuid=uuid, is_known=is_known, ) ) - # Batch recognition + # Batched recognition for tracks queued this frame if need_recog and self.arc is not None: - log.debug(f"Running recognition for {len(need_recog)} tracks") + log.debug("Running recognition for %d tracks", len(need_recog)) self._run_recognition_batch(frame, need_recog, now) elif need_recog: - log.warning(f"need_recog={len(need_recog)} but arc is None") + log.warning("need_recog=%d but arc is None", len(need_recog)) - # Cleanup stale tracks + # Cleanup tracks that BoTSORT stopped reporting self._cleanup_stale() - # Build faces sorted by bbox area (largest = closest first) + # Build the by-area sorted face list used by /who and unknown capture + self._current_faces = self._build_current_faces(results) + + return results + + # ------------------------------------------------------------------ + # Re-identify / display + # ------------------------------------------------------------------ + + def _maybe_reset_for_reidentify(self, ident: _TrackIdentity, now: float) -> None: + """Reset votes if it's been a while since last recognition attempt. + + Two trigger cases: + - status == "unknown": maybe the face has gotten clearer since. + - status == "identified" but tier == "tentative" or sim is marginal: + try to upgrade to "confident" when conditions improve. + + Interaction with AutoEnroller + ----------------------------- + While ``status == "unknown"`` (between finalize and re-identify + firing), no recognition runs for this track, so AutoEnroller stops + receiving observe() calls. This is acceptable: AutoEnroller's buffer + from before finalization is still valid (samples don't age inside + the buffer), and re-identify will resume observations within + ``re_identify_interval`` seconds (default 3s) — well before + AutoEnroller's ``stale_track_sec`` (default 5s) would GC the buffer. + """ + if self.re_identify_interval <= 0: + return + if (now - ident.last_recog_time) < self.re_identify_interval: + return + + should_reset = False + if ident.status == "unknown": + should_reset = True + log.debug( + "Track %d: re-identify (was unknown, %.1fs since last attempt)", + ident.track_id, + now - ident.last_recog_time, + ) + elif ident.status == "identified" and ident.tier == sl.TIER_TENTATIVE: + should_reset = True + log.debug( + "Track %d: re-verify '%s' (tentative, sim=%.3f)", + ident.track_id, + ident.name, + ident.sim, + ) + elif ident.status == "identified" and ident.sim < self.sim_thr + 0.1: + # Confident but marginal sim — re-verify when face gets clearer. + should_reset = True + log.debug( + "Track %d: re-verify '%s' (marginal sim=%.3f)", + ident.track_id, + ident.name, + ident.sim, + ) + + if should_reset: + ident.status = "new" + ident.vote_names.clear() + ident.vote_uuids.clear() + ident.vote_tiers.clear() + ident.vote_sims.clear() + ident.embeddings.clear() + ident.recog_attempts = 0 + + @staticmethod + def _anon_label(uuid: Optional[str]) -> str: + """Short, stable, human-readable label for an anonymous (un-named) UUID. + + Example: '73d0a41a63aa4594...' → 'anon_73d0a4' + + The 6 hex chars give ~16.7M possible labels — collision-free for any + realistic gallery size. The label is deterministic: same UUID always + maps to the same label, even across restarts. The LLM can recognize + this as "we've met but I don't know your name yet" and prompt for one. + """ + if not uuid: + return "unknown" + return f"anon_{uuid[:6]}" + + def _build_display( + self, ident: _TrackIdentity + ) -> Tuple[Optional[str], Optional[str], bool, float, str, Optional[str]]: + """Return ``(raw_name, display_name, is_known, sim, tier, uuid)``. + + ``raw_name`` is the plain label for downstream consumers (no score). + ``display_name`` is the formatted string for on-frame drawing. + ``is_known`` is True only when tier is confident. + ``uuid`` is the gallery UUID for stable identity references (None + when no identification has been made yet). + """ + # Confident identification with a name + if ( + ident.status == "identified" + and ident.tier == sl.TIER_CONFIDENT + and ident.name + ): + return ( + ident.name, + f"{ident.name} ({ident.sim:.2f})", + True, + ident.sim, + sl.TIER_CONFIDENT, + ident.uuid, + ) + + # Confident identification but the UUID has no name yet (auto-enrolled, + # awaiting /set_name). Surface a stable short label so the LLM and + # operator can distinguish "we've met before" from "complete stranger". + if ( + ident.status == "identified" + and ident.tier == sl.TIER_CONFIDENT + and not ident.name + and ident.uuid + ): + label = self._anon_label(ident.uuid) + return ( + label, + f"{label} ({ident.sim:.2f})", + True, + ident.sim, + sl.TIER_CONFIDENT, + ident.uuid, + ) + + # Tentative identification with a name — show with "?" suffix + if ( + ident.status == "identified" + and ident.tier == sl.TIER_TENTATIVE + and ident.name + ): + return ( + ident.name, + f"{ident.name}? ({ident.sim:.2f})", + False, + ident.sim, + sl.TIER_TENTATIVE, + ident.uuid, + ) + + # Tentative identification for an anonymous UUID (matched something + # in gallery but with shaky confidence, and that UUID has no name). + if ( + ident.status == "identified" + and ident.tier == sl.TIER_TENTATIVE + and not ident.name + and ident.uuid + ): + label = self._anon_label(ident.uuid) + return ( + label, + f"{label}? ({ident.sim:.2f})", + False, + ident.sim, + sl.TIER_TENTATIVE, + ident.uuid, + ) + + # Voting still in progress — surface a tentative best guess if we have one + if ident.status == "identifying" and ident.vote_names: + real = [ + (n, u, s) + for n, u, t, s in zip( + ident.vote_names, + ident.vote_uuids, + ident.vote_tiers, + ident.vote_sims, + ) + if n != "unknown" + ] + if real: + top_name = max( + {n for n, _, _ in real}, + key=lambda x: sum(1 for n, _, _ in real if n == x), + ) + top_sim = max(s for n, _, s in real if n == top_name) + # Best-guess UUID = most-frequent UUID among votes for top_name + cand_uuids = [u for n, u, _ in real if n == top_name and u is not None] + top_uuid = ( + Counter(cand_uuids).most_common(1)[0][0] if cand_uuids else None + ) + # Anonymous UUID (no name yet) → use stable short label + display_label = top_name if top_name else self._anon_label(top_uuid) + return ( + display_label, + f"{display_label}? ({top_sim:.2f})", + False, + top_sim, + sl.TIER_TENTATIVE, + top_uuid, + ) + + # New or unknown — no identity available + return (None, "unknown", False, 0.0, sl.TIER_UNCERTAIN, None) + + def _build_current_faces(self, results: List[TrackResult]) -> list: + """Sort the per-track results into a by-area face list. + + Used by /who and the auto-enroll candidate selector. "Largest face" + is a cheap proxy for "closest to camera = most likely the addressee". + """ faces = [] for r in results: x1, y1, x2, y2 = r.bbox area = (x2 - x1) * (y2 - y1) + # Look up the snapshotted session-start last_seen for this + # track. Falls back to None for tracks not yet confidently + # identified (e.g. "unknown" status, or first few frames). ident = self._identities.get(r.track_id) - name = ident.name if ident and ident.status == "identified" else "unknown" + session_last_seen = ident.session_last_seen_iso if ident else None faces.append( { - "name": name, + "name": r.raw_name if r.raw_name else "unknown", + "uuid": r.uuid, + "tier": r.tier, "bbox": r.bbox, "area": area, "track_id": r.track_id, + "session_last_seen_iso": session_last_seen, } ) - self._current_faces = sorted(faces, key=lambda f: f["area"], reverse=True) - - return results + return sorted(faces, key=lambda f: f["area"], reverse=True) - def _get_display_name( - self, ident: _TrackIdentity - ) -> Tuple[Optional[str], bool, float]: - """Get display name for a track based on its identity status. - - Returns (name_display, is_known, sim). - """ - if ident.status == "identified" and ident.name: - # Confirmed identity - return f"{ident.name} ({ident.sim:.2f})", True, ident.sim - - elif ident.status == "identifying" and ident.vote_names: - # Voting in progress — show temporary best guess with "?" - real = [n for n in ident.vote_names if n != "__unknown__"] - if real: - top_name = max(set(real), key=real.count) - top_sim = max( - s - for n, s in zip(ident.vote_names, ident.vote_sims) - if n == top_name - ) - return f"{top_name}? ({top_sim:.2f})", False, top_sim - return "unknown", False, 0.0 - - elif ident.status == "unknown": - return "unknown", False, 0.0 - - else: - # New track, no recognition yet - return "unknown", False, 0.0 + # ------------------------------------------------------------------ + # BoTSORT plumbing + # ------------------------------------------------------------------ def _run_tracker(self, dets: np.ndarray, frame: np.ndarray) -> np.ndarray: """Run BoTSORT. Returns (M, 7) [x1,y1,x2,y2,track_id,conf,cls].""" @@ -367,9 +809,9 @@ def _match_tracks_to_dets( ) -> Dict[int, dict]: """Match tracked boxes back to SCRFD detections to recover landmarks. - Returns {track_id: {"det_idx": int, "kps": ndarray(5,2) or None}} + Returns ``{track_id: {"det_idx": int, "kps": ndarray(5,2) or None}}``. """ - result = {} + result: Dict[int, dict] = {} if tracks is None or len(tracks) == 0 or dets is None or len(dets) == 0: return result @@ -397,7 +839,7 @@ def _match_tracks_to_dets( best_d = d_idx if best_d >= 0 and best_iou > 0.3: - entry: dict[str, object] = {"det_idx": best_d} + entry: dict = {"det_idx": best_d} entry["kps"] = ( kpss[best_d] if kpss is not None and best_d < len(kpss) else None ) @@ -405,22 +847,26 @@ def _match_tracks_to_dets( return result + # ------------------------------------------------------------------ # Recognition + # ------------------------------------------------------------------ def _run_recognition_batch( self, frame: np.ndarray, - need_recog: List[Tuple[int, np.ndarray, Optional[np.ndarray]]], + need_recog: List[Tuple[int, np.ndarray, Optional[np.ndarray], float]], now: float, ) -> None: - """Run AdaFace on unidentified tracks and update votes.""" + """Embed unidentified tracks in one TRT call and update their votes.""" from .adaface import warp_face_by_5p crops: List[np.ndarray] = [] crop_track_ids: List[int] = [] crop_bboxes: List[np.ndarray] = [] + crop_kpss: List[Optional[np.ndarray]] = [] + crop_confs: List[float] = [] - for track_id, bbox, kps in need_recog: + for track_id, bbox, kps, conf in need_recog: try: x1, y1, x2, y2 = bbox if kps is not None: @@ -433,13 +879,15 @@ def _run_recognition_batch( crops.append(crop) crop_track_ids.append(track_id) crop_bboxes.append(bbox) + crop_kpss.append(kps) + crop_confs.append(conf) except Exception: continue if not crops: return - # Batch inference + # Batched AdaFace inference all_feats = [] for i in range(0, len(crops), self.arc_max_bs): batch = crops[i : i + self.arc_max_bs] @@ -452,13 +900,38 @@ def _run_recognition_batch( np.concatenate(all_feats, axis=0) if len(all_feats) > 1 else all_feats[0] ) - # Match against gallery if self._gal_feats is None or self._gal_feats.size == 0: + # Even with empty gallery, we want to give the auto-enroller a + # chance to observe (everyone is "uncertain" in this state, but + # we still need the embedding-buffering path to fire). + for idx, track_id in enumerate(crop_track_ids): + ident = self._identities.get(track_id) + if ident is None: + continue + ident.last_recog_time = now + ident.recog_attempts += 1 + ident.status = "identifying" + ident.embeddings.append(feats[idx]) + ident.vote_names.append("unknown") + ident.vote_uuids.append(None) + ident.vote_tiers.append(sl.TIER_UNCERTAIN) + ident.vote_sims.append(0.0) + self._fire_unknown_callback( + track_id, + crops[idx], + feats[idx], + tuple(crop_bboxes[idx]), + sl.TIER_UNCERTAIN, + ) + if ( + len(ident.vote_names) >= self.vote_frames + or ident.recog_attempts >= self.max_recog_attempts + ): + self._finalize_identity(ident) return + # Single matmul vs. gallery centroids: (B, dim) @ (dim, N_id) = (B, N_id) S = feats @ self._gal_feats.T - best_j = np.argmax(S, axis=1) - best_v = S[np.arange(S.shape[0]), best_j] for idx, track_id in enumerate(crop_track_ids): ident = self._identities.get(track_id) @@ -469,122 +942,314 @@ def _run_recognition_batch( ident.recog_attempts += 1 ident.status = "identifying" - sim_val = float(best_v[idx]) - label_idx = int(best_j[idx]) + name, tier, s1, s2 = sl.recognize_from_sims( + S[idx], + self._gal_labels, + min_sim=self.sim_thr, + strict_margin=self.strict_margin, + loose_margin=self.loose_margin, + ) + + # Map name → UUID using the top-1 row of S. recognize_from_sims + # returns the name of the best-scoring row; for tier "uncertain" + # the name is "unknown" so there's no UUID to attach. + if name != "unknown" and self._gal_uuids: + top_idx = int(np.argmax(S[idx])) + vote_uuid: Optional[str] = self._gal_uuids[top_idx] + else: + vote_uuid = None ident.embeddings.append(feats[idx]) + ident.vote_names.append(name) + ident.vote_uuids.append(vote_uuid) + ident.vote_tiers.append(tier) + ident.vote_sims.append(s1) log.debug( - f"Track {ident.track_id} bbox={crop_bboxes[idx][:4]}: sim={sim_val:.3f} best='{self._gal_labels[label_idx]}' → " - f"{'MATCH' if sim_val >= self.sim_thr else 'NO MATCH'}" + "Track %d bbox=%s: top1=%s/%.3f top2=%.3f margin=%.3f → tier=%s uuid=%s", + track_id, + tuple(crop_bboxes[idx]), + name, + s1, + s2, + s1 - s2, + tier, + vote_uuid[:8] + "..." if vote_uuid else "-", ) - if sim_val >= self.sim_thr: - ident.vote_names.append(self._gal_labels[label_idx]) - ident.vote_sims.append(sim_val) - else: - ident.vote_names.append("__unknown__") - ident.vote_sims.append(sim_val) - # Check if ready to decide + # Notify auto-enroller (it decides what to do based on tier) + self._fire_unknown_callback( + track_id, + crops[idx], + feats[idx], + tuple(crop_bboxes[idx]), + tier, + crop_kpss[idx], + crop_confs[idx], + ) + + # Notify refresh manager on confident identifications. Refresh + # manager applies its own guards (rate-limit, diversity) before + # actually folding the sample into the gallery. We pass the + # specific UUID this frame matched (vote_uuid) — could differ + # from the track's finalized UUID if the track's voting hasn't + # converged yet, but for confident per-frame matches the + # difference is rarely material. + if ( + tier == sl.TIER_CONFIDENT + and vote_uuid + and self.on_confident_sample is not None + ): + try: + self.on_confident_sample( + track_id, + vote_uuid, + crops[idx], + feats[idx], + s1, + ) + except Exception as e: + log.warning("on_confident_sample callback raised: %s", e) + + # Decide as soon as we have enough votes or hit the cap if ( len(ident.vote_names) >= self.vote_frames or ident.recog_attempts >= self.max_recog_attempts ): self._finalize_identity(ident) + def _fire_unknown_callback( + self, + track_id: int, + crop: np.ndarray, + embedding: np.ndarray, + bbox: Tuple[int, int, int, int], + tier: str, + kps: Optional[np.ndarray] = None, + conf: float = 1.0, + ) -> None: + """Forward a recognition observation to the auto-enroll callback. + + The callback receives every tier (not just uncertain) — AutoEnroller + uses non-uncertain observations to clear its per-track buffer when a + formerly-unknown face becomes identified. + """ + if self.on_unknown_sample is None: + return + try: + self.on_unknown_sample(track_id, crop, embedding, bbox, tier, kps, conf) + except Exception as e: + log.warning("on_unknown_sample callback raised: %s", e) + def _finalize_identity(self, ident: _TrackIdentity) -> None: - """Decide identity from collected votes using majority voting.""" + """Decide a track's identity from collected (name, tier) votes. + + Strategy: + - Tally CONFIDENT votes by name. + - Tally TENTATIVE votes by name (these EXCLUDE confident; they're a + separate weaker bucket). + - If any name's confident-fraction >= vote_threshold → identified + confident. + - Else if any name's (confident + tentative)-fraction >= threshold + → identified tentative. + - Else → unknown. + + This means a track with mixed evidence (some confident, some tentative + for the same name) is preferentially called confident — confident + votes are strictly stronger than tentative ones. + """ now = time.time() + total_votes = len(ident.vote_names) - if not ident.vote_names: - ident.status = "unknown" - ident.name = None - if ident.unknown_since == 0: - ident.unknown_since = now + if total_votes == 0: + self._set_unknown(ident, now, reason="no_votes") return - real_votes = [n for n in ident.vote_names if n != "__unknown__"] + confident_counts: Counter = Counter() + loose_counts: Counter = Counter() # confident + tentative for each name + for name, tier in zip(ident.vote_names, ident.vote_tiers): + if name == "unknown": + continue + if tier == sl.TIER_CONFIDENT: + confident_counts[name] += 1 + loose_counts[name] += 1 + elif tier == sl.TIER_TENTATIVE: + loose_counts[name] += 1 + # tier == "uncertain" never carries a real name (name == "unknown") + + # Try confident first + if confident_counts: + best_name, best_count = confident_counts.most_common(1)[0] + if best_count / total_votes >= self.vote_threshold: + self._set_identified( + ident, + best_name, + sl.TIER_CONFIDENT, + now, + votes_for_name=best_count, + total_votes=total_votes, + ) + return + + # Fall back to tentative (still requires the same vote_threshold of + # loose evidence — we just don't demand confident-tier evidence) + if loose_counts: + best_name, best_count = loose_counts.most_common(1)[0] + if best_count / total_votes >= self.vote_threshold: + self._set_identified( + ident, + best_name, + sl.TIER_TENTATIVE, + now, + votes_for_name=best_count, + total_votes=total_votes, + ) + return - if not real_votes: - ident.status = "unknown" - ident.name = None - if ident.unknown_since == 0: - ident.unknown_since = now - log.debug(f"Track {ident.track_id}: all votes unknown") - return + self._set_unknown(ident, now, reason="no_majority") - counter = Counter(real_votes) - best_name, best_count = counter.most_common(1)[0] - total_votes = len(ident.vote_names) - ratio = best_count / total_votes + def _set_identified( + self, + ident: _TrackIdentity, + name: str, + tier: str, + now: float, + *, + votes_for_name: int, + total_votes: int, + ) -> None: + """Apply an identified decision to a track and log it.""" + winning_sims = [ + s for n, s in zip(ident.vote_names, ident.vote_sims) if n == name + ] + avg_sim = sum(winning_sims) / len(winning_sims) if winning_sims else 0.0 + + # Pick the UUID associated with the winning name. Most-frequent UUID + # among the votes that voted for `name` — robust to the rare case + # where the same name maps to two distinct UUIDs over the vote window + # (e.g. two enrollments of the same person; both centroids matched at + # different votes). + winning_uuids = [ + u + for n, u in zip(ident.vote_names, ident.vote_uuids) + if n == name and u is not None + ] + winning_uuid: Optional[str] = None + if winning_uuids: + winning_uuid = Counter(winning_uuids).most_common(1)[0][0] + + ident.status = "identified" + ident.tier = tier + ident.name = name + ident.uuid = winning_uuid + ident.sim = avg_sim + ident.unknown_since = ( + 0.0 if tier == sl.TIER_CONFIDENT else (ident.unknown_since or now) + ) - if ratio >= self.vote_threshold: - winning_sims = [ - s for n, s in zip(ident.vote_names, ident.vote_sims) if n == best_name - ] - avg_sim = sum(winning_sims) / len(winning_sims) if winning_sims else 0.0 + # --- identity-flip detection (auto-merge trigger) --- + # If this same continuous track was previously CONFIDENT on a + # different UUID, the two UUIDs are very likely the same person whose + # poses got separate enrollments (frontal vs profile). Fire the flip + # callback so the outer layer can merge them. Only confident->confident + # transitions count; we reuse the tier already decided above. + if tier == sl.TIER_CONFIDENT and winning_uuid: + prev = ident.last_confident_uuid + if prev and prev != winning_uuid and self.on_identity_flip is not None: + try: + self.on_identity_flip(ident.track_id, prev, winning_uuid) + except Exception as e: + log.warning("on_identity_flip callback raised: %s", e) + ident.last_confident_uuid = winning_uuid - ident.status = "identified" - ident.name = best_name - ident.sim = avg_sim - ident.unknown_since = 0 # no longer unknown - log.info( - f"Track {ident.track_id}: confirmed '{best_name}' " - f"(votes={best_count}/{total_votes}, sim={avg_sim:.3f})" - ) - else: - ident.status = "unknown" - ident.name = None - if ident.unknown_since == 0: - ident.unknown_since = now - log.debug( - f"Track {ident.track_id}: no majority ({best_name} {best_count}/{total_votes})" - ) + log.info( + "Track %d: %s '%s' uuid=%s (votes=%d/%d, sim=%.3f)", + ident.track_id, + tier, + name, + (winning_uuid[:8] + "...") if winning_uuid else "-", + votes_for_name, + total_votes, + avg_sim, + ) + + def _set_unknown(self, ident: _TrackIdentity, now: float, *, reason: str) -> None: + """Mark a track as unknown (no majority found).""" + ident.status = "unknown" + ident.tier = sl.TIER_UNCERTAIN + ident.name = None + ident.uuid = None + if ident.unknown_since == 0: + ident.unknown_since = now + log.debug("Track %d: unknown (%s)", ident.track_id, reason) + + # ------------------------------------------------------------------ + # Cleanup / introspection + # ------------------------------------------------------------------ def _cleanup_stale(self) -> None: - """Remove identity entries for tracks no longer active.""" + """Drop identity state for tracks BoTSORT no longer reports.""" stale = [tid for tid in self._identities if tid not in self._active_ids] for tid in stale: del self._identities[tid] def get_track_identities(self) -> Dict[int, dict]: - """Get current identity state for all active tracks.""" + """Snapshot of current identity state per active track (for /who etc).""" return { tid: { "name": ident.name, + "uuid": ident.uuid, "status": ident.status, + "tier": ident.tier, "sim": ident.sim, "frames_seen": ident.frames_seen, "recog_attempts": ident.recog_attempts, "votes": len(ident.vote_names), + "session_last_seen_iso": ident.session_last_seen_iso, } for tid, ident in self._identities.items() if tid in self._active_ids } + def maybe_set_session_last_seen(self, track_id: int, iso: Optional[str]) -> bool: + """If this track has no ``session_last_seen_iso`` yet, set it. + + Called by the on_confident_sample wiring in run.py the first time + a track is confidently identified, BEFORE touching the gallery's + last_seen_at. After that, subsequent confident matches in the same + track session preserve the snapshotted value (sticky). + + Returns True if set this call, False if already set (or unknown + track). + """ + ident = self._identities.get(track_id) + if ident is None: + return False + if ident.session_last_seen_iso is not None: + return False + ident.session_last_seen_iso = iso + return True + + def get_session_last_seen(self, track_id: int) -> Optional[str]: + """Read the sticky session-start last_seen for a track, or None.""" + ident = self._identities.get(track_id) + if ident is None: + return None + return ident.session_last_seen_iso + def get_faces(self) -> list: - """Get all faces sorted by bbox area (largest first). + """All faces in the current frame, sorted by bbox area descending. - Returns - ------- - list of dict - Each dict has 'name', 'bbox', 'area', 'track_id'. - Empty list if no faces in current frame. + Each entry: ``{name, tier, bbox, area, track_id}``. + ``name`` is ``"unknown"`` if the track has no identification (or is + only tentative — callers check ``tier`` separately). """ return self._current_faces def get_unknowns(self) -> list: - """Get all unknown faces in current frame with duration info. - - Returns - ------- - list of dict - Each dict has 'track_id', 'bbox', 'area', 'unknown_duration'. - Empty list if no unknowns. - """ + """Faces currently unidentified, with how long they've been unknown.""" now = time.time() unknowns = [] - for face in self._current_faces: if face["name"] != "unknown": continue @@ -592,7 +1257,6 @@ def get_unknowns(self) -> list: ident = self._identities.get(tid) if ident is None or ident.unknown_since <= 0: continue - unknowns.append( { "track_id": tid, @@ -601,11 +1265,10 @@ def get_unknowns(self) -> list: "unknown_duration": round(now - ident.unknown_since, 1), } ) - return unknowns def reset(self) -> None: - """Reset all tracking state.""" + """Reset all tracking state (e.g. after gallery wipe between venues).""" self._identities.clear() self._active_ids.clear() self._current_faces.clear() diff --git a/src/om1_vlm/anonymizationSys/face_recog_stream/gallery.py b/src/om1_vlm/anonymizationSys/face_recog_stream/gallery.py index 92c1a48..118bc44 100644 --- a/src/om1_vlm/anonymizationSys/face_recog_stream/gallery.py +++ b/src/om1_vlm/anonymizationSys/face_recog_stream/gallery.py @@ -1,23 +1,48 @@ """ -Gallery & embedding manager for face recognition. - -This module manages a disk-backed face gallery and its embedding store. -It aligns new photos (optionally from `raw/` using SCRFD) into 112×112 -`aligned/` crops, embeds them with AdaFace (TensorRT), and persists the -results under `embeds//{index.json,vectors.f32,stats.json}`. -It supports incremental refresh (only process new files), adding single -aligned snapshots, clearing & rebuilding, and computing per-identity mean -vectors for fast runtime recognition. - -Typical usage: -- At startup (or after HTTP actions like /gallery/refresh, /gallery/add_aligned, - /gallery/add_raw, /selfie), refresh the gallery and fetch identity means. -- At inference time, compare a face embedding to the returned per-identity means. - -Notes ------ -- Embeddings are float32; vectors are L2-normalized before similarity. -- Batch size respects the AdaFace TensorRT optimization profile (`arc_max_bs`). +UUID-keyed face gallery with per-identity self-contained folders. + +Layout +------ +gallery/ + / + metadata.json # {uuid, name, created_at, updated_at, sample_count, + # source, notes?} + aligned/ # 112x112 BGR JPGs + sample_001.jpg + sample_002.jpg + ... + embeddings.npy # (k, 512) float32, L2-normalized, one row per sample + centroid.npy # (512,) float32, L2-normalized incremental mean + _legacy/ # one-shot migration moves old / folders here + _trash/ # forget()'d UUIDs (soft delete, restorable) + +Why UUIDs not names +------------------- +- Renames are O(1): just update metadata.name (no folder rename, no GPU work, + no embedding store rewrite). +- Multiple "profiles" per name are natural: two distinct UUIDs can both be + named "wendy" (e.g. one auto-enrolled, one selfie). Either can be deleted + independently. +- Auto-enrolled identities start with name="" (anonymous) and get named + later when the LLM asks. The UUID never changes — only the metadata. +- Privacy: filenames don't leak identities; the UUID is opaque on disk. + metadata.json is the only place names live, and can be encrypted later. + +Why per-folder embeddings (vs one global file) +---------------------------------------------- +- forget(uuid) is rmtree on one folder — O(1) filesystem op, no GPU. + The old design rebuilt the entire embedding store on every delete, which + was ~25 seconds at 1000 identities × 5 samples. +- add_sample updates one centroid incrementally — O(1), no scan of others. +- get_centroids() scans folders at load and caches; in-memory after that. + +Threading model +--------------- +All gallery write operations should be called from a single thread (the +main loop, via run_job_sync). This module does NOT take internal locks — +the caller is responsible for serialization. Reads (get_centroids, +list_identities) are safe to call from any thread but should ideally also +be serialized with writes. """ from __future__ import annotations @@ -26,19 +51,33 @@ import logging import os import os.path as osp +import re import shutil import time +import uuid as uuidlib +from dataclasses import dataclass, field from typing import Dict, List, Optional, Tuple import cv2 import numpy as np -from .adaface import TRTFaceRecognition, warp_face_by_5p -from .scrfd import TRTSCRFD -from .utils import infer_arc_batched, list_images # helpers +from .adaface import TRTFaceRecognition log = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) + +# UUIDs are written as plain hex strings (no dashes) to keep filenames short. +# Regex matches both lowercase hex strings and standard 8-4-4-4-12 format. +_UUID_RE = re.compile( + r"^[a-f0-9]{32}$|^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$" +) + +# Reserved directory names — never treated as UUIDs. +_RESERVED_DIRS = frozenset({"_legacy", "_trash", "embeds"}) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- def _ensure_dir(d: str) -> None: @@ -46,654 +85,1171 @@ def _ensure_dir(d: str) -> None: def _now_iso() -> str: - return time.strftime("%Y-%m-%dT%H-%M-%S") + return time.strftime("%Y-%m-%dT%H:%M:%S") -def make_model_sig(arc: TRTFaceRecognition, extra: Optional[str] = None) -> str: - """Build a readable, deterministic model signature for the embedding store. +def _is_uuid_dir(name: str) -> bool: + return bool(_UUID_RE.match(name)) - Parameters - ---------- - arc : TRTFaceRecognition - AdaFace TensorRT wrapper; used to read model name and embedding dimension. - extra : str, optional - Extra token to append (e.g., engine variant), by default ``None``. - - Returns - ------- - str - Signature string like ``"{name}-trt-l2-{dim}[-{extra}]"``. + +def _atomic_write_json(path: str, obj: dict) -> None: + """Write JSON via temp-file + rename so partial writes can't corrupt. + + Crashes between the write and rename leave the previous file intact. """ - name = getattr(arc, "name", None) or getattr(arc, "model_name", None) or "arc" - dim = int(getattr(arc, "embedding_dim", 512)) - backend = "trt" - norm = "l2" - parts = [str(name), backend, norm, str(dim)] - if extra: - parts.append(str(extra)) - return "-".join(parts) - - -def _align_largest_face_bgr( - img_bgr: np.ndarray, - det: TRTSCRFD, - conf: float, - size: int = 112, -) -> Optional[np.ndarray]: + _ensure_dir(osp.dirname(path)) + tmp = f"{path}.tmp.{os.getpid()}" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(obj, f, ensure_ascii=False, indent=2) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + + +def _atomic_write_npy(path: str, arr: np.ndarray) -> None: + """Write .npy via temp + rename. Same crash-safety as JSON above.""" + _ensure_dir(osp.dirname(path)) + tmp = f"{path}.tmp.{os.getpid()}" + np.save(tmp, arr.astype(np.float32, copy=False)) + # np.save appends .npy automatically if extension missing — handle that. + if not osp.exists(tmp) and osp.exists(tmp + ".npy"): + tmp = tmp + ".npy" + os.replace(tmp, path) + + +def _l2_normalize(v: np.ndarray) -> np.ndarray: + """L2-normalize, defensive against zero vectors.""" + n = float(np.linalg.norm(v)) + if n < 1e-12: + return v.astype(np.float32, copy=False) + return (v / n).astype(np.float32, copy=False) + + +def _new_uuid(existing: set) -> str: + """Generate a fresh UUID4 (hex form, 32 chars). + + UUID4 collision is astronomically unlikely (~10^-15 at 100M UUIDs), but + we retry on the off chance to be safe — also catches the case where a + user manually copied a folder. """ - Detect faces, choose the largest, align with 5p if available, else crop+resize. - Returns a size×size BGR crop or None if no face. + for _ in range(10): + u = uuidlib.uuid4().hex + if u not in existing: + return u + raise RuntimeError("UUID generation collision (10x). Bug or corrupt state.") + + +# --------------------------------------------------------------------------- +# Per-UUID record (in-memory representation) +# --------------------------------------------------------------------------- + + +@dataclass +class IdentityRecord: + """In-memory snapshot of one UUID folder. + + Fields are populated from metadata.json + centroid.npy + the aligned/ dir. + Mutations on this object are NOT written back automatically — call + ``UUIDGallery.save_metadata(record)`` after mutating ``record.metadata``. + + Centroid representation + ----------------------- + ``centroid_sum`` is the UN-normalized sum of all sample embeddings — i.e. + ``sum(embeddings)``, not ``mean(embeddings)`` and not L2-normalized. This + is what we persist in ``centroid.npy``. + + Why store the sum (not the L2-normalized mean): + Adding a new sample is just ``sum_new = sum_old + emb``. If we stored the + L2-normalized mean, we'd lose the magnitude information needed to weight + the existing mean correctly when folding in a new vector. Storing the + sum keeps the add operation O(1) and mathematically equivalent to a + full recompute. + + The L2-normalized form (what face recognition actually uses) is computed + on the fly in :meth:`UUIDGallery.get_centroids` — that's just N vector + normalizations, trivial even at thousands of identities. """ - dets, kpss = det.detect(img_bgr, conf=conf) - if dets is None or dets.shape[0] == 0: - return None - areas = (dets[:, 2] - dets[:, 0]) * (dets[:, 3] - dets[:, 1]) - idx = int(np.argmax(areas)) - if kpss is not None: - try: - return warp_face_by_5p(img_bgr, kpss[idx], size) - except Exception: - pass - x1, y1, x2, y2, _ = dets[idx].astype(int) - face = img_bgr[max(0, y1) : max(0, y2), max(0, x1) : max(0, x2)] - if face.size == 0: - return None - return cv2.resize(face, (size, size)) + uuid: str + name: str + sample_count: int + centroid_sum: Optional[np.ndarray] = None # un-normalized sum, (512,) + metadata: dict = field(default_factory=dict) + + @property + def is_named(self) -> bool: + """True if this identity has a non-empty display name (i.e. not anon).""" + return bool(self.name and self.name.strip()) + + @property + def centroid_normalized(self) -> Optional[np.ndarray]: + """Lazy L2-normalized centroid for similarity comparisons.""" + if self.centroid_sum is None: + return None + return _l2_normalize(self.centroid_sum) -class GalleryManager: - """ - Gallery manager for face recognition. - - Manages a face gallery with aligned crops and an embedding store. It supports incremental refresh from - ``raw/`` and ``aligned/``, batched embedding respecting the AdaFace TensorRT max batch size, and per-identity statistics (counts and mean vectors) - - Layout: - gallery/ - Alice/ aligned/*.jpg (112x112) [source of truth] - raw/*.jpg (optional; will be aligned -> aligned/] - Bob/ aligned/*.jpg - embeds/ - / - index.json { "meta": {...}, "items": { "Alice/aligned/foo.jpg": {"row":0,"label":"Alice"}, ... } } - vectors.f32 float32 raw, concatenated (N * dim) - stats.json { "labels": {"Alice": {"count": X, "mean": [...]}}, "count": N, "dim": dim } - - Typical usage: - gm = GalleryManager(gallery_dir, embeds_dir, arc, scrfd) - gm.refresh() # incremental: align new raw/, embed new aligned/ - feats, labels = gm.get_identity_means() # per-identity means (for recognition) - - Attributes + +# --------------------------------------------------------------------------- +# UUIDGallery +# --------------------------------------------------------------------------- + + +class UUIDGallery: + """Disk-backed face gallery keyed by UUID. + + Parameters ---------- gallery_dir : str - Absolute path to the gallery root (one subfolder per identity). - embeds_root : str - Absolute path to the embeddings root folder. + Root directory containing the UUID folders. arc : TRTFaceRecognition - AdaFace TensorRT wrapper used for feature extraction. - scrfd : TRTSCRFD or None - SCRFD detector used to align new images from ``raw/``. - det_conf : float - Detection confidence threshold for alignment. - aligned_size : int - Aligned face crop size (pixels). - model_sig : str - Model signature used to segregate embed stores. - arc_max_bs : int - Maximum AdaFace batch size (matches TRT optimization profile). - store_dir : str - Directory under ``embeds_root`` for this model signature. - index_path : str - JSON index mapping relative image paths to row indices and labels. - vectors_path : str - Binary file storing concatenated float32 embeddings ``(N * dim)``. - stats_path : str - JSON file with per-identity counts and mean vectors. + AdaFace TRT engine for embedding extraction. Used by ``embed_aligned`` + and any operation that adds a sample without a pre-computed vector. + aligned_size : int, default 112 + Face crop dimension. Almost always 112 for AdaFace/ArcFace. """ def __init__( self, gallery_dir: str, - embeds_dir: str, *, arc: TRTFaceRecognition, - scrfd: Optional[TRTSCRFD] = None, - det_conf: float = 0.5, aligned_size: int = 112, - model_sig: Optional[str] = None, - arc_max_bs: int = 4, # IMPORTANT: respect TensorRT optimization profile (e.g., 1..4) + max_samples_per_uuid: int = 50, + last_seen_persist_interval_sec: float = 60.0, ) -> None: - """ - Initialize the gallery manager and load/create the embed store. - """ self.gallery_dir = osp.abspath(gallery_dir) - self.embeds_root = osp.abspath(embeds_dir) self.arc = arc - self.scrfd = scrfd - self.det_conf = float(det_conf) self.aligned_size = int(aligned_size) - self.model_sig = model_sig or make_model_sig(arc) - self.arc_max_bs = int(arc_max_bs) - - # Paths under embeds/ - self.store_dir = osp.join(self.embeds_root, self.model_sig) - self.index_path = osp.join(self.store_dir, "index.json") - self.vectors_path = osp.join(self.store_dir, "vectors.f32") - self.stats_path = osp.join(self.store_dir, "stats.json") - - _ensure_dir(self.store_dir) - self._index = self._load_index() - self._dim = int(getattr(self.arc, "embedding_dim", 512)) - - def refresh(self, process_raw: bool = True) -> Tuple[int, int]: + self._dim = int(getattr(arc, "embedding_dim", 512)) + # FIFO cap per UUID. When add_sample would push past this, the + # oldest sample (lexicographically first JPG, row 0 of embeddings.npy) + # is evicted: file removed, embedding subtracted from centroid_sum, + # row 0 of embeddings.npy dropped. Keeps disk bounded as the system + # accumulates samples over long-running deployments and from the + # continuous-refresh path. + self.max_samples_per_uuid = int(max_samples_per_uuid) + + # touch_last_seen is called every confident-recognition frame + # (~15 Hz × N visible faces). Writing metadata.json that often + # would dominate disk I/O. Instead, we maintain a two-layer + # cache: rec.metadata["last_seen_at"] is updated in memory on + # every call (cheap), and metadata.json is flushed to disk at + # most once per this many seconds per UUID. The disk lag is + # bounded; if the process crashes, we lose at most this many + # seconds of last_seen accuracy — fine for the /gallery/sweep_stale + # use case (cleanup thresholds are typically days/months). + self.last_seen_persist_interval_sec = float(last_seen_persist_interval_sec) + # Per-UUID monotonic timestamp of the most recent metadata.json + # write triggered by touch_last_seen. Used to enforce the + # rate-limit above. + self._last_seen_disk_ts: Dict[str, float] = {} + + _ensure_dir(self.gallery_dir) + _ensure_dir(osp.join(self.gallery_dir, "_trash")) + + # In-memory cache: uuid → IdentityRecord. Populated on demand by + # _load() and refreshed by reload(). + self._records: Dict[str, IdentityRecord] = {} + self.reload() + + # ================================================================== + # Loading + # ================================================================== + + def reload(self) -> int: + """Scan ``gallery_dir`` and rebuild the in-memory record cache. + + Returns the number of UUIDs loaded. Call this after external + modifications to ``gallery_dir`` (e.g. file-level rsync). """ - Incrementally update the embed store. - - - Optionally process new images in `raw/`: detect→align→write into `aligned/`. - - Embed any new 112×112 crops under `aligned/` and append to vectors. - - Parameters - ---------- - process_raw : bool, optional - Whether to detect→align images from ``raw/`` into ``aligned/``, by default ``True``. + self._records.clear() + self._last_seen_disk_ts.clear() + now_mono = time.monotonic() + if not osp.isdir(self.gallery_dir): + return 0 + for entry in os.listdir(self.gallery_dir): + if entry in _RESERVED_DIRS: + continue + path = osp.join(self.gallery_dir, entry) + if not osp.isdir(path): + continue + if not _is_uuid_dir(entry): + # Legacy folder; needs migration via migrate_legacy(). + log.warning( + "Skipping non-UUID directory '%s' (run migrate_legacy)", + entry, + ) + continue + try: + rec = self._load(entry) + if rec is not None: + self._records[entry] = rec + # Seed the rate-limit tracker: the file on disk is + # current at load time, so touch_last_seen shouldn't + # immediately re-write it. The first disk write for + # this UUID won't happen until persist_interval seconds + # have passed. + self._last_seen_disk_ts[entry] = now_mono + except Exception as e: + log.warning("Failed to load UUID %s: %s", entry, e) + log.info("Gallery loaded: %d UUIDs", len(self._records)) + return len(self._records) + + def _load(self, uuid: str) -> Optional[IdentityRecord]: + """Load one UUID's record from disk. Returns None on corrupted entry.""" + uuid_dir = osp.join(self.gallery_dir, uuid) + meta_path = osp.join(uuid_dir, "metadata.json") + centroid_path = osp.join(uuid_dir, "centroid.npy") + + if not osp.exists(meta_path): + log.warning("UUID %s missing metadata.json", uuid) + return None + try: + with open(meta_path, "r", encoding="utf-8") as f: + meta = json.load(f) + except Exception as e: + log.warning("UUID %s metadata.json invalid: %s", uuid, e) + return None - Returns - ------- - (num_aligned_added, num_vectors_appended) - """ - if process_raw and self.scrfd is None: - log.warning( - "refresh(): 'process_raw=True' but SCRFD was not provided; skipping raw/" - ) + centroid_sum: Optional[np.ndarray] = None + if osp.exists(centroid_path): + try: + centroid_sum = np.load(centroid_path).astype(np.float32) + if centroid_sum is not None and centroid_sum.shape != (self._dim,): + log.warning( + "UUID %s centroid shape %s, expected (%d,)", + uuid, + centroid_sum.shape, + self._dim, + ) + centroid_sum = None + except Exception as e: + log.warning("UUID %s centroid.npy invalid: %s", uuid, e) + + sample_count = int(meta.get("sample_count", 0)) + # Cross-check sample_count against actual files (cheap sanity check) + aligned_dir = osp.join(uuid_dir, "aligned") + if osp.isdir(aligned_dir): + actual = len([f for f in os.listdir(aligned_dir) if f.endswith(".jpg")]) + if actual != sample_count: + log.debug( + "UUID %s: metadata sample_count=%d, actual files=%d (will fix)", + uuid, + sample_count, + actual, + ) + sample_count = actual + meta["sample_count"] = actual + + return IdentityRecord( + uuid=uuid, + name=str(meta.get("name", "")), + sample_count=sample_count, + centroid_sum=centroid_sum, + metadata=meta, + ) - if process_raw and self.scrfd is not None: - self._harvest_raw_to_aligned() + # ================================================================== + # Read APIs (called from recognition + UI) + # ================================================================== - added = self._embed_new_aligned() - self._recompute_stats() - return added + def get_centroids(self) -> Tuple[np.ndarray, List[str], List[str]]: + """Build the (feats, names, uuids) triple consumed by face_tracker. - def clear_and_rebuild(self) -> Tuple[int, int]: - """Delete current vectors and rebuild embeddings from ``aligned/`` only. + Only includes UUIDs with at least one sample. Names are the + human-readable labels from metadata.json — they may be empty for + auto-enrolled (yet-to-be-named) identities. Returns ------- - tuple[int, int] - ``(num_aligned_added, num_vectors_appended)`` after rebuild. + feats : (N, 512) float32 + L2-normalized centroids, one row per included UUID. + names : list[str] + Display names, parallel to ``feats``. + uuids : list[str] + UUIDs, parallel to ``feats``. Use this for stable identity refs + (e.g. /set_name, /forget). """ - if osp.exists(self.vectors_path): - try: - os.remove(self.vectors_path) - except Exception: - pass - self._index = {"meta": self._index.get("meta", {}), "items": {}} - self._save_index() - - added = self._embed_new_aligned() - self._recompute_stats() - return added - - def add_aligned_snapshot( - self, label: str, img_112_bgr: np.ndarray, fname_hint: Optional[str] = None - ) -> str: + feats_list: List[np.ndarray] = [] + names: List[str] = [] + uuids: List[str] = [] + for u, rec in self._records.items(): + if rec.centroid_sum is None or rec.sample_count == 0: + continue + # Normalize the running sum for cosine-similarity use. Storing + # the un-normalized sum on disk lets add_sample stay O(1); the + # per-frame normalization here is N vector norms (trivial). + feats_list.append(_l2_normalize(rec.centroid_sum)) + names.append(rec.name) + uuids.append(u) + if not feats_list: + return np.zeros((0, self._dim), dtype=np.float32), [], [] + return np.stack(feats_list, axis=0).astype(np.float32), names, uuids + + def get_record(self, uuid: str) -> Optional[IdentityRecord]: + """Look up a single UUID's record.""" + return self._records.get(uuid) + + def get_metadata(self, uuid: str) -> Optional[dict]: + """Return metadata.json contents for ``uuid``, or None if unknown.""" + rec = self._records.get(uuid) + return dict(rec.metadata) if rec is not None else None + + def list_identities(self, include_unnamed: bool = True) -> List[dict]: + """Public listing of all identities (one dict per UUID). + + Used by /gallery/identities. Returns lightweight summary dicts — + not full centroids. """ - Add a new 112×112 face (already aligned) under gallery/