Detection infrastructure rework: Secondary detection stream, In-process LiteRT engine, Configurable grace period

.gitmodules

@@ -2,3 +2,6 @@
 	path = go2rtc
 	url = https://github.com/opensensor/go2rtc.git
 	branch = dev
+[submodule "third_party/litert"]
+	path = third_party/litert
+	url = https://github.com/google-ai-edge/LiteRT.git

CMakeLists.txt

@@ -10,6 +10,14 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 option(ENABLE_SOD "Enable SOD library for object detection" ON)
 option(SOD_DYNAMIC_LINK "Dynamically link SOD library instead of static linking" OFF)
 
+# In-process LiteRT (TFLite) detection engine. Default ON, but auto-disabled at
+# configure time if the third_party/litert submodule isn't initialized, so a
+# fresh checkout still builds. When ON (and present), the vendored LiteRT
+# submodule is built and linked, and the litert_engine module is compiled.
+option(ENABLE_LITERT "Enable in-process LiteRT (TFLite) detection engine" ON)
+option(LITERT_WITH_XNNPACK "Compile XNNPACK delegate into the LiteRT engine" ON)
+option(LITERT_WITH_GPU "Compile GPU (OpenCL/GLES) delegate into the LiteRT engine" OFF)
+
 # go2rtc integration options
 option(ENABLE_GO2RTC "Enable go2rtc integration for WebRTC streaming" ON)
 set(GO2RTC_BINARY_PATH "/usr/local/bin/go2rtc" CACHE STRING "Path to go2rtc binary")
@@ -246,6 +254,60 @@ if(ENABLE_SOD)
     endif()
 endif()
 
+# Set up LiteRT (in-process TFLite engine) if enabled. Auto-disable it when the
+# submodule isn't initialized so a default configure still succeeds out-of-the-box.
+set(LITERT_ENGINE_SOURCES "")
+if(ENABLE_LITERT AND NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/third_party/litert/tflite/CMakeLists.txt")
+    message(WARNING
+        "ENABLE_LITERT is ON but third_party/litert is not initialized; "
+        "disabling the in-process LiteRT engine. Run "
+        "'git submodule update --init --recursive third_party/litert' to enable it.")
+    set(ENABLE_LITERT OFF)
+endif()
+if(ENABLE_LITERT)
+    set(LITERT_SUBDIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/litert/tflite")
+
+    # Forward delegate selections to the LiteRT subproject.
+    set(TFLITE_ENABLE_XNNPACK ${LITERT_WITH_XNNPACK} CACHE BOOL "" FORCE)
+    set(TFLITE_ENABLE_GPU ${LITERT_WITH_GPU} CACHE BOOL "" FORCE)
+    # ruy is TFLite's ARM CPU GEMM backend; it runs any ops the XNNPACK delegate
+    # doesn't handle (falling back to it instead of the slower gemmlowp/Eigen path).
+    set(TFLITE_ENABLE_RUY ON CACHE BOOL "" FORCE)
+    set(TFLITE_BUILD_SHARED_LIB OFF CACHE BOOL "" FORCE)
+
+    # Build XNNPACK without its hand-written assembly microkernels, so it uses the
+    # portable NEON-intrinsic kernels instead. XNNPACK's cortex_a53 asm f32 6x8 GEMM
+    # kernel corrupts an A-row pointer under a multi-threaded pthreadpool on aarch64
+    # big.LITTLE, SIGSEGV-ing with threads>1 (root-caused from a core dump; present
+    # even at XNNPACK HEAD). Disabling the asm path makes multi-threaded inference
+    # stable; confirmed on target. Small single-thread perf cost vs tuned asm.
+    if(LITERT_WITH_XNNPACK)
+        set(XNNPACK_ENABLE_ASSEMBLY OFF CACHE BOOL "" FORCE)
+    endif()
+
+    add_subdirectory(${LITERT_SUBDIR} EXCLUDE_FROM_ALL)
+    add_compile_definitions(HAVE_LITERT)
+
+    if(LITERT_WITH_XNNPACK)
+        add_compile_definitions(HAVE_LITERT_XNNPACK)
+    endif()
+    if(LITERT_WITH_GPU)
+        add_compile_definitions(HAVE_LITERT_GPU)
+    endif()
+
+    set(LITERT_ENGINE_SOURCES
+        "${CMAKE_CURRENT_SOURCE_DIR}/src/video/detection/litert_engine.cc"
+    )
+    # TFLite public headers (and the Abseil/FlatBuffers they pull in) require
+    # C++17, but the project default is C++14. Linking the tensorflow-lite
+    # target propagates cxx_std_17, yet make the requirement explicit on our
+    # own engine source so it compiles correctly regardless of propagation.
+    set_source_files_properties(${LITERT_ENGINE_SOURCES} PROPERTIES CXX_STANDARD 17)
+    message(STATUS "LiteRT engine enabled (XNNPACK=${LITERT_WITH_XNNPACK}, GPU=${LITERT_WITH_GPU})")
+else()
+    message(STATUS "LiteRT engine disabled (set -DENABLE_LITERT=ON to enable)")
+endif()
+
 # HTTP Backend Configuration - libuv + llhttp only
 message(STATUS "Using libuv + llhttp HTTP backend")
 add_definitions(-DHTTP_BACKEND_LIBUV)
@@ -471,6 +533,7 @@ set(SOURCES
         ${HLS_UNIFIED_THREAD_SOURCES}
         ${TELEMETRY_SOURCES}
         ${EZXML_SOURCES}
+        ${LITERT_ENGINE_SOURCES}
 )
 
 
@@ -581,6 +644,14 @@ if(ENABLE_MQTT AND MOSQUITTO_FOUND)
     target_include_directories(lightnvr PRIVATE ${MOSQUITTO_INCLUDE_DIRS})
 endif()
 
+# Link LiteRT if enabled
+if(ENABLE_LITERT)
+    # LiteRT's CMake target is named tensorflow-lite (provided by the LiteRT
+    # submodule's tflite/CMakeLists.txt).
+    target_link_libraries(lightnvr_lib PUBLIC tensorflow-lite)
+    target_link_libraries(lightnvr tensorflow-lite)
+endif()
+
 # Link SOD library if enabled
 if(ENABLE_SOD)
     # Always link to the sod target, whether it's built as static or shared
@@ -683,6 +754,11 @@ if(ENABLE_SOD)
         message(STATUS "  - SOD linking method: Static")
     endif()
 endif()
+message(STATUS "- LiteRT (in-process TFLite engine): ${ENABLE_LITERT}")
+if(ENABLE_LITERT)
+    message(STATUS "  - XNNPACK delegate: ${LITERT_WITH_XNNPACK}")
+    message(STATUS "  - GPU delegate: ${LITERT_WITH_GPU}")
+endif()
 message(STATUS "- go2rtc WebRTC integration: ${ENABLE_GO2RTC}")
 if(ENABLE_GO2RTC)
     message(STATUS "  - go2rtc binary path: ${GO2RTC_BINARY_PATH}")

config/lightnvr.ini

@@ -49,6 +49,15 @@ backend = onnx  ; Detection backend: onnx (YOLOv8 - best accuracy), tflite, or o
 confidence_threshold = 0.35  ; Lower threshold to catch distant vehicles
 filter_classes = car,motorcycle,truck,bus,bicycle  ; Vehicle classes only
 
+[detection_engine]
+; In-process TFLite/LiteRT inference. When enabled, any stream whose
+; model_path ends in .tflite will route through this engine. Requires
+; LightNVR built with -DENABLE_LITERT=ON. Labels are read from a sidecar
+; <basename>.labels.txt next to the model.
+enabled = false
+threads = 1         ; Interpreter threads (1-16); raise to use multiple CPU cores
+delegate = xnnpack  ; xnnpack | gpu | none (gpu requires -DLITERT_WITH_GPU=ON; falls back to CPU otherwise)
+
 [memory]
 buffer_size = 1024  ; Buffer size in KB
 use_swap = true

db/migrations/0042_add_detection_url.sql

@@ -0,0 +1,19 @@
+-- Add detection_url to streams table
+--
+-- Optional secondary stream URL used exclusively for object detection
+-- (e.g. a low-resolution MJPEG sub-stream).  When non-empty the Unified
+-- Detection Thread opens this URL in a dedicated background thread and runs
+-- the configured detection model against its frames, while the main stream
+-- URL is used only for buffering and MP4 recording.
+--
+-- MJPEG streams are ideal because every frame is a full JPEG keyframe,
+-- removing the large inter-keyframe gap that limits detection accuracy on
+-- the primary H.264/H.265 RTSP stream.
+
+-- migrate:up
+
+ALTER TABLE streams ADD COLUMN detection_url TEXT DEFAULT '';
+
+-- migrate:down
+
+SELECT 1;

include/core/config.h

@@ -115,11 +115,28 @@ typedef struct {
     // When non-empty, registered with go2rtc as "{name}_sub" and used by the
     // frontend in grid view while the main URL is used for fullscreen/recording.
     char sub_stream_url[MAX_URL_LENGTH];
+
+    // Detection stream URL: optional secondary stream used exclusively for
+    // object detection (e.g. an MJPEG sub-stream).  When non-empty the UDT
+    // opens this URL in a dedicated background thread and runs the configured
+    // detection model against its frames; the main stream is used only for
+    // buffering and recording.  MJPEG is ideal here because every frame is a
+    // full keyframe, eliminating the large inter-frame gap that limits detection
+    // accuracy on the main H.264/H.265 RTSP stream.
+    char detection_url[MAX_URL_LENGTH];
 } stream_config_t;
 
 // Size of recording schedule text buffer: 168 values + 167 commas + null terminator
 #define RECORDING_SCHEDULE_TEXT_SIZE 512
 
+// In-process LiteRT (TFLite) detection engine — runtime knobs only.
+// Per-model state (model_path, labels) is configured per-stream, not here.
+typedef struct {
+    bool enabled;       // Master toggle (default: false)
+    int  num_threads;   // Interpreter::SetNumThreads, clamped 1..16 (default: 1)
+    char delegate[16];  // "xnnpack" | "gpu" | "none" (default: "xnnpack")
+} detection_engine_config_t;
+
 // Main configuration structure
 typedef struct {
     // General settings
@@ -161,6 +178,7 @@ typedef struct {
     int default_pre_detection_buffer;      // Default seconds to keep before detection (0-60)
     int default_post_detection_buffer;     // Default seconds to keep after detection (0-300)
     char default_buffer_strategy[32];      // Default buffer strategy: auto, go2rtc, hls_segment, memory_packet, mmap_hybrid
+    int detection_grace_period;            // Seconds after last detection before entering post-buffer (default: 2)
 
     // Database settings
     char db_path[MAX_PATH_LENGTH];
@@ -261,6 +279,10 @@ typedef struct {
     bool mqtt_ha_discovery;               // Enable HA MQTT auto-discovery (default: false)
     char mqtt_ha_discovery_prefix[128];   // HA discovery topic prefix (default: "homeassistant")
     int mqtt_ha_snapshot_interval;        // Snapshot publish interval in seconds (default: 30, 0=disabled)
+
+    // In-process LiteRT (TFLite) detection engine.
+    // Runtime knobs only; per-model state (path, labels) lives elsewhere.
+    detection_engine_config_t detection_engine;
 } config_t;
 
 /**
@@ -275,15 +297,6 @@ typedef struct {
  */
 int load_config(config_t *config);
 
-/**
- * Reload configuration from disk
- * This is used to refresh the global config after settings changes
- * 
- * @param config Pointer to config structure to fill
- * @return 0 on success, non-zero on failure
- */
-int reload_config(config_t *config);
-
 /**
  * Save configuration to specified file
  * 
@@ -300,6 +313,20 @@ int save_config(const config_t *config, const char *path);
  */
 void load_default_config(config_t *config);
 
+/**
+ * Set the detection grace period, clamped to its valid range (0..60 s).
+ * Shared by the config-file loader and the settings API so the range lives
+ * in one place.
+ */
+void config_set_detection_grace_period(config_t *config, int seconds);
+
+/** Set the LiteRT interpreter thread count, clamped to 1..16. */
+void config_set_detection_engine_threads(config_t *config, int threads);
+
+/** Set the LiteRT delegate if it is one of "xnnpack"/"gpu"/"none"; returns
+ *  true if accepted, false (leaving the config unchanged) otherwise. */
+bool config_set_detection_engine_delegate(config_t *config, const char *delegate);
+
 /**
  * Validate and normalize configuration values
  * 

include/database/db_embedded_migrations.h

@@ -636,6 +636,12 @@ static const char migration_0041_up[] =
 static const char migration_0041_down[] =
     "SELECT 1;";
 
+static const char migration_0042_up[] =
+    "ALTER TABLE streams ADD COLUMN detection_url TEXT DEFAULT '';";
+
+static const char migration_0042_down[] =
+    "SELECT 1;";
+
 static const migration_t embedded_migrations_data[] = {
     {
         .version = "0001",
@@ -924,8 +930,15 @@ static const migration_t embedded_migrations_data[] = {
         .sql_down = migration_0041_down,
         .is_embedded = true
     },
+    {
+        .version = "0042",
+        .description = "add_detection_url",
+        .sql_up = migration_0042_up,
+        .sql_down = migration_0042_down,
+        .is_embedded = true
+    },
 };
 
-#define EMBEDDED_MIGRATIONS_COUNT 41
+#define EMBEDDED_MIGRATIONS_COUNT 42
 
 #endif /* DB_EMBEDDED_MIGRATIONS_H */

include/video/api_detection.h

@@ -3,6 +3,7 @@
 
 #include <stdbool.h>
 #include <stdint.h>
+#include <time.h>
 #include "video/detection_result.h"
 
 // Model type for API-based detection
@@ -36,11 +37,16 @@ void shutdown_api_detection_system(void);
  * @param stream_name The name of the stream (for database storage)
  * @param threshold Confidence threshold for detection (0.0-1.0, use negative for default)
  * @param recording_id Recording ID to link detections to (0 for no link)
+ * @param frame_timestamp Wall-clock time when the frame entered the detection
+ *                       pipeline. Used as the DB/MQTT event timestamp so it
+ *                       reflects frame time rather than inference completion.
+ *                       Pass 0 to fall back to time(NULL) inside the DB layer.
  * @return 0 on success, non-zero on failure
  */
 int detect_objects_api(const char *api_url, const unsigned char *frame_data,
                       int width, int height, int channels, detection_result_t *result,
-                      const char *stream_name, float threshold, uint64_t recording_id);
+                      const char *stream_name, float threshold, uint64_t recording_id,
+                      time_t frame_timestamp);
 
 /**
  * @brief Determine whether API detection should fetch a go2rtc snapshot.
@@ -69,6 +75,7 @@ bool api_detection_should_use_go2rtc_snapshot(const unsigned char *frame_data,
  * @return 0 on success, -1 on general failure, -2 if go2rtc snapshot failed (caller should fall back)
  */
 int detect_objects_api_snapshot(const char *api_url, const char *stream_name,
-                                detection_result_t *result, float threshold, uint64_t recording_id);
+                                detection_result_t *result, float threshold,
+                                uint64_t recording_id, time_t frame_timestamp);
 
 #endif /* LIGHTNVR_API_DETECTION_H */

include/video/detection/litert_engine.h

@@ -0,0 +1,83 @@
+/**
+ * litert_engine.h
+ *
+ * C ABI for the in-process LiteRT (TFLite) detection engine.
+ *
+ * Consumed only by src/video/detection_model.c and src/video/detection.c.
+ * Implementation in litert_engine.cc.
+ *
+ * All entry points are no-ops returning NULL/-1 when ENABLE_LITERT=OFF
+ * (HAVE_LITERT undefined). Callers must still gate calls with
+ * #ifdef HAVE_LITERT to avoid unresolved-symbol linkage.
+ */
+#ifndef LITERT_ENGINE_H
+#define LITERT_ENGINE_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#include "video/detection_result.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct litert_engine litert_engine_t;
+
+/* Acquire (or create) a refcounted engine for the given model file.
+ * Multiple acquires of the same canonical path return the same engine,
+ * with refcount bumped. Returns NULL on failure (errors logged).
+ *
+ * Failure modes (each logs a clear message before returning NULL):
+ *   - g_config.detection_engine.enabled == false
+ *   - model file missing or unreadable
+ *   - flatbuffer parse failure
+ *   - unsupported input/output dtype (only float32/uint8/int8)
+ *   - unsupported output shape (must be [1, N, 6])
+ *   - delegate ModifyGraphWithDelegate failure (after CPU fallback attempt)
+ */
+litert_engine_t *litert_engine_acquire(const char *model_path);
+
+/* Decrement refcount; when it reaches 0, destroy the engine and release
+ * delegate resources. Safe to call with NULL. */
+void litert_engine_release(litert_engine_t *engine);
+
+/* Model input dimensions, as read from the .tflite at init time.
+ * Returns 0 if engine is NULL. */
+int litert_engine_input_width(const litert_engine_t *engine);
+int litert_engine_input_height(const litert_engine_t *engine);
+
+/* Combined size of the loaded model flatbuffers (weights + graph) across all
+ * engines — the detector's resident footprint for the system memory stats.
+ * The tensor arena is deliberately excluded: it can't be measured accurately
+ * here and over-counting it made the reported figure exceed process RSS.
+ * Returns 0 when ENABLE_LITERT=OFF or no engines are loaded. Thread-safe. */
+uint64_t litert_engine_registry_memory_bytes(void);
+
+/* Run inference on a single RGB24 frame (packed HxWx3 in original frame
+ * coordinates). The engine letterboxes internally to its model HxW.
+ *
+ * Detection count is written to out->count. Boxes are in
+ * out->detections[*].x/y/width/height as normalized 0..1 in the original
+ * frame coordinate space. Labels are read from a <basename>.labels.txt
+ * sidecar next to the model, falling back to "class_<n>".
+ *
+ * per_stream_threshold is the only confidence cutoff (no engine-side
+ * floor). Range 0.0..1.0; the engine internally clamps to [0, 1].
+ *
+ * Returns 0 on success (matching detect_objects() / detect_with_sod_model()
+ * convention — non-zero is treated as failure by the UDT), or -1 on
+ * error. On success out->count is 0 when no objects pass the threshold.
+ *
+ * Thread-safe — each engine has its own mutex; concurrent calls
+ * serialize through it. */
+int litert_engine_detect(litert_engine_t *engine,
+                         const uint8_t *rgb, int w, int h,
+                         float per_stream_threshold,
+                         detection_result_t *out);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* LITERT_ENGINE_H */