From a464197dcc82ae60f7eecdb372c05dfdec70354b Mon Sep 17 00:00:00 2001 From: Piotr Ziecik Date: Fri, 19 Jun 2026 10:38:22 +0200 Subject: [PATCH 01/13] feat(detection): store optional detection_url per stream Add a per-stream detection_url column: an optional secondary stream (e.g. a low-res MJPEG sub-stream) used exclusively for object detection, leaving the main stream for buffering/recording. - migration 0042 adds the detection_url TEXT column (filesystem + embedded) - db_streams.c loads/saves it alongside the other stream fields - stream_config_t gains the in-memory detection_url field Co-Authored-By: Claude Opus 4.8 (1M context) --- db/migrations/0042_add_detection_url.sql | 19 +++++++++++ include/core/config.h | 9 +++++ include/database/db_embedded_migrations.h | 15 ++++++++- src/database/db_streams.c | 41 +++++++++++++++++------ 4 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 db/migrations/0042_add_detection_url.sql diff --git a/db/migrations/0042_add_detection_url.sql b/db/migrations/0042_add_detection_url.sql new file mode 100644 index 00000000..3fe7285c --- /dev/null +++ b/db/migrations/0042_add_detection_url.sql @@ -0,0 +1,19 @@ +-- Add detection_url to streams table +-- +-- Optional secondary stream URL used exclusively for object detection +-- (e.g. a low-resolution MJPEG sub-stream). When non-empty the Unified +-- Detection Thread opens this URL in a dedicated background thread and runs +-- the configured detection model against its frames, while the main stream +-- URL is used only for buffering and MP4 recording. +-- +-- MJPEG streams are ideal because every frame is a full JPEG keyframe, +-- removing the large inter-keyframe gap that limits detection accuracy on +-- the primary H.264/H.265 RTSP stream. + +-- migrate:up + +ALTER TABLE streams ADD COLUMN detection_url TEXT DEFAULT ''; + +-- migrate:down + +SELECT 1; diff --git a/include/core/config.h b/include/core/config.h index 4abaea90..6908a5b8 100644 --- a/include/core/config.h +++ b/include/core/config.h @@ -115,6 +115,15 @@ typedef struct { // When non-empty, registered with go2rtc as "{name}_sub" and used by the // frontend in grid view while the main URL is used for fullscreen/recording. char sub_stream_url[MAX_URL_LENGTH]; + + // Detection stream URL: optional secondary stream used exclusively for + // object detection (e.g. an MJPEG sub-stream). When non-empty the UDT + // opens this URL in a dedicated background thread and runs the configured + // detection model against its frames; the main stream is used only for + // buffering and recording. MJPEG is ideal here because every frame is a + // full keyframe, eliminating the large inter-frame gap that limits detection + // accuracy on the main H.264/H.265 RTSP stream. + char detection_url[MAX_URL_LENGTH]; } stream_config_t; // Size of recording schedule text buffer: 168 values + 167 commas + null terminator diff --git a/include/database/db_embedded_migrations.h b/include/database/db_embedded_migrations.h index 94b7c95c..d7f6a5ac 100644 --- a/include/database/db_embedded_migrations.h +++ b/include/database/db_embedded_migrations.h @@ -636,6 +636,12 @@ static const char migration_0041_up[] = static const char migration_0041_down[] = "SELECT 1;"; +static const char migration_0042_up[] = + "ALTER TABLE streams ADD COLUMN detection_url TEXT DEFAULT '';"; + +static const char migration_0042_down[] = + "SELECT 1;"; + static const migration_t embedded_migrations_data[] = { { .version = "0001", @@ -924,8 +930,15 @@ static const migration_t embedded_migrations_data[] = { .sql_down = migration_0041_down, .is_embedded = true }, + { + .version = "0042", + .description = "add_detection_url", + .sql_up = migration_0042_up, + .sql_down = migration_0042_down, + .is_embedded = true + }, }; -#define EMBEDDED_MIGRATIONS_COUNT 41 +#define EMBEDDED_MIGRATIONS_COUNT 42 #endif /* DB_EMBEDDED_MIGRATIONS_H */ diff --git a/src/database/db_streams.c b/src/database/db_streams.c index f8d88605..fe14b86d 100644 --- a/src/database/db_streams.c +++ b/src/database/db_streams.c @@ -133,7 +133,8 @@ uint64_t add_stream_config(const stream_config_t *stream) { "onvif_username = ?, onvif_password = ?, onvif_profile = ?, onvif_port = ?, " "record_on_schedule = ?, recording_schedule = ?, tags = ?, admin_url = ?, " "privacy_mode = ?, motion_trigger_source = ?, go2rtc_source_override = ?, " - "sub_stream_url = ?, audio_voice_enhancement = ? " + "sub_stream_url = ?, audio_voice_enhancement = ?, " + "detection_url = ? " "WHERE id = ?;"; rc = sqlite3_prepare_v2(db, update_sql, -1, &stmt, NULL); @@ -218,9 +219,10 @@ uint64_t add_stream_config(const stream_config_t *stream) { sqlite3_bind_text(stmt, 46, stream->go2rtc_source_override, -1, SQLITE_STATIC); sqlite3_bind_text(stmt, 47, stream->sub_stream_url, -1, SQLITE_STATIC); sqlite3_bind_int(stmt, 48, stream->audio_voice_enhancement ? 1 : 0); + sqlite3_bind_text(stmt, 49, stream->detection_url, -1, SQLITE_STATIC); // Bind ID parameter - sqlite3_bind_int64(stmt, 49, (sqlite3_int64)existing_id); + sqlite3_bind_int64(stmt, 50, (sqlite3_int64)existing_id); // Execute statement rc = sqlite3_step(stmt); @@ -269,8 +271,8 @@ uint64_t add_stream_config(const stream_config_t *stream) { "ptz_enabled, ptz_max_x, ptz_max_y, ptz_max_z, ptz_has_home, " "onvif_username, onvif_password, onvif_profile, onvif_port, " "record_on_schedule, recording_schedule, tags, admin_url, privacy_mode, motion_trigger_source, " - "go2rtc_source_override, sub_stream_url, audio_voice_enhancement) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; + "go2rtc_source_override, sub_stream_url, audio_voice_enhancement, detection_url) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); if (rc != SQLITE_OK) { @@ -355,6 +357,7 @@ uint64_t add_stream_config(const stream_config_t *stream) { sqlite3_bind_text(stmt, 47, stream->go2rtc_source_override, -1, SQLITE_STATIC); sqlite3_bind_text(stmt, 48, stream->sub_stream_url, -1, SQLITE_STATIC); sqlite3_bind_int(stmt, 49, stream->audio_voice_enhancement ? 1 : 0); + sqlite3_bind_text(stmt, 50, stream->detection_url, -1, SQLITE_STATIC); // Execute statement rc = sqlite3_step(stmt); @@ -426,7 +429,8 @@ int update_stream_config(const char *name, const stream_config_t *stream) { "onvif_username = ?, onvif_password = ?, onvif_profile = ?, onvif_port = ?, " "record_on_schedule = ?, recording_schedule = ?, tags = ?, admin_url = ?, privacy_mode = ?, " "motion_trigger_source = ?, go2rtc_source_override = ?, " - "sub_stream_url = ?, audio_voice_enhancement = ? " + "sub_stream_url = ?, audio_voice_enhancement = ?, " + "detection_url = ? " "WHERE name = ?;"; rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); @@ -512,9 +516,10 @@ int update_stream_config(const char *name, const stream_config_t *stream) { sqlite3_bind_text(stmt, 47, stream->go2rtc_source_override, -1, SQLITE_STATIC); sqlite3_bind_text(stmt, 48, stream->sub_stream_url, -1, SQLITE_STATIC); sqlite3_bind_int(stmt, 49, stream->audio_voice_enhancement ? 1 : 0); + sqlite3_bind_text(stmt, 50, stream->detection_url, -1, SQLITE_STATIC); // Bind the WHERE clause parameter - sqlite3_bind_text(stmt, 50, name, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 51, name, -1, SQLITE_STATIC); // Execute statement rc = sqlite3_step(stmt); @@ -787,7 +792,7 @@ int get_stream_config_by_name(const char *name, stream_config_t *stream) { "ptz_enabled, ptz_max_x, ptz_max_y, ptz_max_z, ptz_has_home, " "onvif_username, onvif_password, onvif_profile, onvif_port, " "record_on_schedule, recording_schedule, tags, admin_url, privacy_mode, motion_trigger_source, " - "go2rtc_source_override, sub_stream_url, audio_voice_enhancement " + "go2rtc_source_override, sub_stream_url, audio_voice_enhancement, detection_url " "FROM streams WHERE name = ?;"; // Column index constants for readability @@ -804,7 +809,7 @@ int get_stream_config_by_name(const char *name, stream_config_t *stream) { COL_ONVIF_USERNAME, COL_ONVIF_PASSWORD, COL_ONVIF_PROFILE, COL_ONVIF_PORT, COL_RECORD_ON_SCHEDULE, COL_RECORDING_SCHEDULE, COL_TAGS, COL_ADMIN_URL, COL_PRIVACY_MODE, COL_MOTION_TRIGGER_SOURCE, COL_GO2RTC_SOURCE_OVERRIDE, COL_SUB_STREAM_URL, - COL_AUDIO_VOICE_ENHANCEMENT + COL_AUDIO_VOICE_ENHANCEMENT, COL_DETECTION_URL }; rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); @@ -984,6 +989,14 @@ int get_stream_config_by_name(const char *name, stream_config_t *stream) { // Audio voice-enhancement opt-in (discussion #395) stream->audio_voice_enhancement = sqlite3_column_int(stmt, COL_AUDIO_VOICE_ENHANCEMENT) != 0; + // Secondary detection stream URL + const char *detection_url_val = (const char *)sqlite3_column_text(stmt, COL_DETECTION_URL); + if (detection_url_val) { + safe_strcpy(stream->detection_url, detection_url_val, sizeof(stream->detection_url), 0); + } else { + stream->detection_url[0] = '\0'; + } + result = 0; } @@ -1036,7 +1049,7 @@ int get_all_stream_configs(stream_config_t *streams, int max_count) { "ptz_enabled, ptz_max_x, ptz_max_y, ptz_max_z, ptz_has_home, " "onvif_username, onvif_password, onvif_profile, onvif_port, " "record_on_schedule, recording_schedule, tags, admin_url, privacy_mode, motion_trigger_source, " - "go2rtc_source_override, sub_stream_url, audio_voice_enhancement " + "go2rtc_source_override, sub_stream_url, audio_voice_enhancement, detection_url " "FROM streams ORDER BY name;"; // Column index constants (same as get_stream_config_by_name) @@ -1053,7 +1066,7 @@ int get_all_stream_configs(stream_config_t *streams, int max_count) { COL_ONVIF_USERNAME, COL_ONVIF_PASSWORD, COL_ONVIF_PROFILE, COL_ONVIF_PORT, COL_RECORD_ON_SCHEDULE, COL_RECORDING_SCHEDULE, COL_TAGS, COL_ADMIN_URL, COL_PRIVACY_MODE, COL_MOTION_TRIGGER_SOURCE, COL_GO2RTC_SOURCE_OVERRIDE, COL_SUB_STREAM_URL, - COL_AUDIO_VOICE_ENHANCEMENT + COL_AUDIO_VOICE_ENHANCEMENT, COL_DETECTION_URL }; rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL); @@ -1232,6 +1245,14 @@ int get_all_stream_configs(stream_config_t *streams, int max_count) { // Audio voice-enhancement opt-in (discussion #395) s->audio_voice_enhancement = sqlite3_column_int(stmt, COL_AUDIO_VOICE_ENHANCEMENT) != 0; + // Secondary detection stream URL + const char *det_url = (const char *)sqlite3_column_text(stmt, COL_DETECTION_URL); + if (det_url) { + safe_strcpy(s->detection_url, det_url, sizeof(s->detection_url), 0); + } else { + s->detection_url[0] = '\0'; + } + count++; } From 4f91aed18d1706a4e15c01b73cfe652250076f40 Mon Sep 17 00:00:00 2001 From: Piotr Ziecik Date: Fri, 19 Jun 2026 10:39:19 +0200 Subject: [PATCH 02/13] feat(api): accept and emit detection_url on stream endpoints - POST/PUT stream handlers parse, validate (http/https/rtsp/rtsps only, blocking file://, concat:, etc.) and persist detection_url, and treat it as a restart-triggering field. - GET stream endpoints emit detection_url so the UI can round-trip it. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/web/api_handlers_streams_get.c | 3 ++ src/web/api_handlers_streams_modify.c | 70 ++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/web/api_handlers_streams_get.c b/src/web/api_handlers_streams_get.c index 0d09d44f..342d831f 100644 --- a/src/web/api_handlers_streams_get.c +++ b/src/web/api_handlers_streams_get.c @@ -239,6 +239,7 @@ void handle_get_streams(const http_request_t *req, http_response_t *res) { cJSON_AddStringToObject(stream_obj, "motion_trigger_source", db_streams[i].motion_trigger_source); cJSON_AddStringToObject(stream_obj, "go2rtc_source_override", db_streams[i].go2rtc_source_override); cJSON_AddStringToObject(stream_obj, "sub_stream_url", db_streams[i].sub_stream_url); + cJSON_AddStringToObject(stream_obj, "detection_url", db_streams[i].detection_url); // Get stream status stream_handle_t stream = get_stream_by_name(db_streams[i].name); @@ -398,6 +399,7 @@ void handle_get_stream(const http_request_t *req, http_response_t *res) { cJSON_AddStringToObject(stream_obj, "motion_trigger_source", config.motion_trigger_source); cJSON_AddStringToObject(stream_obj, "go2rtc_source_override", config.go2rtc_source_override); cJSON_AddStringToObject(stream_obj, "sub_stream_url", config.sub_stream_url); + cJSON_AddStringToObject(stream_obj, "detection_url", config.detection_url); // Get stream status — resolve using UDT state so that go2rtc-managed // streams (which stay INACTIVE in the state manager) report accurately. @@ -551,6 +553,7 @@ void handle_get_stream_full(const http_request_t *req, http_response_t *res) { cJSON_AddStringToObject(stream_obj, "motion_trigger_source", config.motion_trigger_source); cJSON_AddStringToObject(stream_obj, "go2rtc_source_override", config.go2rtc_source_override); cJSON_AddStringToObject(stream_obj, "sub_stream_url", config.sub_stream_url); + cJSON_AddStringToObject(stream_obj, "detection_url", config.detection_url); // Status — resolve using UDT state for accurate reporting when go2rtc // manages the stream (state manager stays INACTIVE/STOPPED at startup). diff --git a/src/web/api_handlers_streams_modify.c b/src/web/api_handlers_streams_modify.c index d6a3d736..a49b60ac 100644 --- a/src/web/api_handlers_streams_modify.c +++ b/src/web/api_handlers_streams_modify.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -54,6 +55,7 @@ typedef struct { bool has_detection_model; // Whether detection_model was provided bool has_detection_threshold; // Whether detection_threshold was provided bool has_detection_interval; // Whether detection_interval was provided + bool has_detection_url; // Whether detection_url was provided bool has_record; // Whether record flag was provided bool has_streaming_enabled; // Whether streaming_enabled flag was provided bool non_dynamic_config_changed; // Whether non-dynamic fields changed @@ -189,7 +191,7 @@ static void put_stream_worker(put_stream_task_t *task) { // we need to restart the stream to apply the new detection settings if (task->config_changed && (task->has_detection_based_recording || task->has_detection_model || - task->has_detection_threshold || task->has_detection_interval) && + task->has_detection_threshold || task->has_detection_interval || task->has_detection_url) && task->is_running && !task->requires_restart) { log_info("Detection settings changed for stream %s, marking for restart to apply changes", task->config.name); task->requires_restart = true; @@ -267,7 +269,7 @@ static void put_stream_worker(put_stream_task_t *task) { } } // If detection settings changed but detection was already enabled, restart the thread - else if (detection_now_enabled && (task->has_detection_model || task->has_detection_threshold || task->has_detection_interval)) { + else if (detection_now_enabled && (task->has_detection_model || task->has_detection_threshold || task->has_detection_interval || task->has_detection_url)) { log_info("Detection settings changed for stream %s, restarting unified detection thread", task->config.name); if (stop_unified_detection_thread(task->config.name) != 0) { @@ -435,6 +437,21 @@ static void put_stream_worker(put_stream_task_t *task) { */ // put_stream_handler removed - work done via update_stream_task_func +/* detection_url is user-supplied and passed to avformat_open_input in the + * detection stream thread. Restrict the scheme to network protocols so + * file://, concat:, subfile:, etc. can't be used to coerce server-side reads. + * An empty string is allowed (means "fall back to main stream"). */ +static bool is_allowed_detection_url(const char *u) { + if (!u || u[0] == '\0') return true; + static const char *const prefixes[] = { + "http://", "https://", "rtsp://", "rtsps://", NULL + }; + for (size_t i = 0; prefixes[i] != NULL; i++) { + if (strncasecmp(u, prefixes[i], strlen(prefixes[i])) == 0) return true; + } + return false; +} + /** * @brief Direct handler for POST /api/streams */ @@ -796,6 +813,23 @@ void handle_post_stream(const http_request_t *req, http_response_t *res) { config.sub_stream_url[0] = '\0'; } + // Parse secondary detection stream URL + cJSON *detection_url_post = cJSON_GetObjectItem(stream_json, "detection_url"); + if (detection_url_post && cJSON_IsString(detection_url_post)) { + if (!is_allowed_detection_url(detection_url_post->valuestring)) { + log_error("Rejected detection_url with disallowed scheme for stream %s", + config.name); + cJSON_Delete(stream_json); + http_response_set_json_error(res, 400, + "detection_url must use http, https, rtsp, or rtsps"); + return; + } + safe_strcpy(config.detection_url, detection_url_post->valuestring, + sizeof(config.detection_url), 0); + } else { + config.detection_url[0] = '\0'; + } + // Check if isOnvif flag is set in the request cJSON *is_onvif = cJSON_GetObjectItem(stream_json, "isOnvif"); if (is_onvif && cJSON_IsBool(is_onvif)) { @@ -1481,6 +1515,37 @@ void handle_put_stream(const http_request_t *req, http_response_t *res) { } } + // Parse secondary detection stream URL + bool has_detection_url = false; + cJSON *detection_url_put = cJSON_GetObjectItem(stream_json, "detection_url"); + if (detection_url_put && cJSON_IsString(detection_url_put)) { + if (!is_allowed_detection_url(detection_url_put->valuestring)) { + log_error("Rejected detection_url with disallowed scheme for stream %s", + config.name); + cJSON_Delete(stream_json); + http_response_set_json_error(res, 400, + "detection_url must use http, https, rtsp, or rtsps"); + return; + } + if (strncmp(config.detection_url, detection_url_put->valuestring, + sizeof(config.detection_url) - 1) != 0) { + safe_strcpy(config.detection_url, detection_url_put->valuestring, + sizeof(config.detection_url), 0); + has_detection_url = true; + config_changed = true; + non_dynamic_config_changed = true; + log_info("Detection URL changed for stream %s", config.name); + } + } else if (detection_url_put && cJSON_IsNull(detection_url_put)) { + if (config.detection_url[0] != '\0') { + config.detection_url[0] = '\0'; + has_detection_url = true; + config_changed = true; + non_dynamic_config_changed = true; + log_info("Detection URL cleared for stream %s", config.name); + } + } + // Update is_onvif flag based on request or URL bool original_is_onvif = config.is_onvif; @@ -1727,6 +1792,7 @@ void handle_put_stream(const http_request_t *req, http_response_t *res) { task->has_detection_model = has_detection_model; task->has_detection_threshold = has_detection_threshold; task->has_detection_interval = has_detection_interval; + task->has_detection_url = has_detection_url; task->has_record = has_record; task->has_streaming_enabled = has_streaming_enabled; task->non_dynamic_config_changed = non_dynamic_config_changed; From e58163cae99b5f5f0977c7ef6167b74737f7f0aa Mon Sep 17 00:00:00 2001 From: Piotr Ziecik Date: Fri, 19 Jun 2026 10:43:16 +0200 Subject: [PATCH 03/13] feat(detection): rework unified detection thread with a secondary detection stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructure the unified detection thread into a clean producer/consumer pipeline and add an optional secondary detection stream: - When a stream's detection_url is set, a dedicated producer thread opens that URL (network protocols only), decodes frames, and runs the configured model; the main thread consumes the result and drives the recording state machine. Detection falls back to the main stream whenever the secondary stream is connecting/reconnecting. - Split the pipeline into detect → report_detections → handle_recording_state. - Intra-only (e.g. MJPEG) detection streams skip redundant decodes via the codec descriptor's INTRA_ONLY property. - current_recording_id is _Atomic for the cross-thread (producer/consumer) read. - Recording robustness: carry pre_buffer_seconds into the MP4 writer and warn on implausible forward DTS jumps before the monotonicity fixup. Co-Authored-By: Claude Opus 4.8 (1M context) --- include/video/mp4_writer.h | 5 + include/video/unified_detection_thread.h | 42 +- src/video/mp4_segment_recorder.c | 27 + src/video/unified_detection_thread.c | 1132 +++++++++++++--------- 4 files changed, 732 insertions(+), 474 deletions(-) diff --git a/include/video/mp4_writer.h b/include/video/mp4_writer.h index f304e191..63cd478c 100644 --- a/include/video/mp4_writer.h +++ b/include/video/mp4_writer.h @@ -64,6 +64,11 @@ struct mp4_writer { // Shutdown coordination int shutdown_component_id; // ID assigned by the shutdown coordinator + // Pre-buffer duration carried in from the detection thread; used by the DTS + // gap detector in mp4_segment_recorder_write_packet() to set its threshold + // (3 × pre_buffer_seconds). 0 means "use fallback (60s)". + int pre_buffer_seconds; + // Pending audio codec parameters set by udt_start_recording() before the first // packet arrives. mp4_writer_initialize() consumes this to declare the audio // stream BEFORE avformat_write_header() is called — the only legal window for diff --git a/include/video/unified_detection_thread.h b/include/video/unified_detection_thread.h index b6ee80c0..52a86390 100644 --- a/include/video/unified_detection_thread.h +++ b/include/video/unified_detection_thread.h @@ -24,6 +24,7 @@ #include "core/config.h" #include "video/packet_buffer.h" #include "video/detection_model.h" +#include "video/detection_result.h" #include "video/mp4_writer.h" #include "video/stream_manager.h" @@ -79,7 +80,11 @@ typedef struct { // MP4 recording mp4_writer_t *mp4_writer; char current_recording_path[MAX_PATH_LENGTH]; - uint64_t current_recording_id; + // Written by the main UDT thread (recording start/stop), read by the + // detection-stream producer thread for API recording_id linkage. + // _Atomic so those cross-thread reads/writes are race-free on all targets + // (notably 32-bit ARM, where a plain 64-bit access is not atomic). + _Atomic uint64_t current_recording_id; // Detection state atomic_llong last_detection_time; // When last detection occurred (stored as atomic epoch seconds) @@ -87,7 +92,7 @@ typedef struct { atomic_llong post_buffer_end_time; // When post-buffer recording should end atomic_int log_counter; // Counter for periodic logging; intentionally accessed without ctx->mutex, // but all accesses must use atomic operations, and exact accuracy is not critical. - + // Connection state atomic_int_fast64_t last_packet_time; atomic_int consecutive_failures; @@ -138,6 +143,39 @@ typedef struct { char onvif_username_cached[64]; char onvif_password_cached[64]; + // ------------------------------------------------------------------------- + // Secondary detection stream thread + // ------------------------------------------------------------------------- + // When detection_url is non-empty a dedicated background thread (pure + // producer) opens that URL (any FFmpeg-readable source), decodes frames, + // and runs the configured detection model. It is responsible ONLY for + // detection inference; all actions (DB storage, recording state) are + // handled by the UDT main loop (consumer). + // + // Producer → consumer protocol (single-slot, overwrite semantics): + // 1. Thread detects → writes result to detection_stream_pending (mutex) + // → sets detection_stream_result = 1 (atomic) + // 2. process_packet reads result via atomic_exchange(0) + mutex + // → calls report_detections + handle_recording_state + // + // detection_stream_connected is set to 1 only once the stream is + // successfully opened and decoding. It is 0 while connecting or + // reconnecting. process_packet uses this — not thread_running — as the + // guard to suppress main-stream detection, so fallback is automatic + // during any outage. + // + // Lifecycle mirrors the ONVIF thread: + // • Created alongside the UDT in start_unified_detection_thread(). + // • Joined inside unified_detection_thread_func() before ctx is freed. + // • Stopped during forced shutdown. + pthread_t detection_stream_thread; + atomic_int detection_stream_thread_running; // 1 = running, 0 = stop requested + atomic_int detection_stream_connected; // 1 = open & decoding, 0 = connecting/disconnected + atomic_int detection_stream_result; // 1 = pending result ready, 0 = none + detection_result_t detection_stream_pending; // latest result; guarded by result_mutex + pthread_mutex_t detection_stream_result_mutex; // protects detection_stream_pending + char detection_stream_url[MAX_URL_LENGTH]; // cached at thread-start + // FFmpeg contexts (managed exclusively by the unified detection thread). // Concurrency / lifetime: // - These pointers and stream indices are created, updated, and freed only diff --git a/src/video/mp4_segment_recorder.c b/src/video/mp4_segment_recorder.c index ccaf9297..3eda1819 100644 --- a/src/video/mp4_segment_recorder.c +++ b/src/video/mp4_segment_recorder.c @@ -1810,6 +1810,33 @@ int mp4_segment_recorder_write_packet(mp4_writer_t *writer, const AVPacket *pkt, out_pkt->pts = out_pkt->dts; } + // Detect implausible forward DTS jumps before the monotonicity fixup + // masks them. A jump larger than 3 × pre_buffer_seconds (fallback 60s) + // means the writer is about to bake a multi-minute gap into the moov — + // exactly the failure mode that produces unplayable detection MP4s. + if (out_pkt->dts != AV_NOPTS_VALUE && writer->last_dts != AV_NOPTS_VALUE && + out_pkt->dts > writer->last_dts) { + int threshold_sec = (writer->pre_buffer_seconds > 0) + ? (3 * writer->pre_buffer_seconds) + : 60; + int64_t threshold_units = av_rescale_q(threshold_sec, + (AVRational){1, 1}, + input_stream->time_base); + int64_t gap_units = out_pkt->dts - writer->last_dts; + if (gap_units > threshold_units) { + double gap_sec = (double)gap_units * av_q2d(input_stream->time_base); + log_warn("[%s] DTS gap detector: %.2fs forward jump in %s " + "(last_dts=%lld new_dts=%lld threshold=%ds) — " + "MP4 timeline will be discontinuous", + writer->stream_name, + gap_sec, + writer->output_path, + (long long)writer->last_dts, + (long long)out_pkt->dts, + threshold_sec); + } + } + // Ensure monotonically increasing DTS values if (out_pkt->dts != AV_NOPTS_VALUE && writer->last_dts != AV_NOPTS_VALUE && out_pkt->dts <= writer->last_dts) { int64_t fixed_dts = writer->last_dts + 1; diff --git a/src/video/unified_detection_thread.c b/src/video/unified_detection_thread.c index 3f94274c..ea9f64fd 100644 --- a/src/video/unified_detection_thread.c +++ b/src/video/unified_detection_thread.c @@ -104,10 +104,22 @@ static bool system_initialized = false; // Forward declarations static void *unified_detection_thread_func(void *arg); +static void *detection_stream_thread_func(void *arg); +static int start_detection_stream_thread(unified_detection_ctx_t *ctx); +static void stop_detection_stream_thread(unified_detection_ctx_t *ctx); +/* Detection layer — pure inference, no DB/state side effects */ +static bool detect_on_decoded_frame(unified_detection_ctx_t *ctx, + AVFrame *frame, time_t now, + detection_result_t *result); +static bool run_detection_on_frame(unified_detection_ctx_t *ctx, AVPacket *pkt); +/* Action layer */ +static void report_detections(unified_detection_ctx_t *ctx, + const detection_result_t *result, time_t now); +static void handle_recording_state(unified_detection_ctx_t *ctx, + bool detected, time_t now); static int connect_to_stream(unified_detection_ctx_t *ctx); static void disconnect_from_stream(unified_detection_ctx_t *ctx); static int process_packet(unified_detection_ctx_t *ctx, AVPacket *pkt); -static bool run_detection_on_frame(unified_detection_ctx_t *ctx, AVPacket *pkt); static int udt_start_recording(unified_detection_ctx_t *ctx); static int udt_stop_recording(unified_detection_ctx_t *ctx); static int flush_prebuffer_to_recording(unified_detection_ctx_t *ctx); @@ -347,6 +359,9 @@ void shutdown_unified_detection_system(void) { stop_onvif_detection_thread(ctx); } + // Stop the detection stream thread (no-op if never started). + stop_detection_stream_thread(ctx); + // Clean up resources if (ctx->packet_buffer) { destroy_packet_buffer(ctx->packet_buffer); @@ -361,9 +376,10 @@ void shutdown_unified_detection_system(void) { ctx->mp4_writer = NULL; } - // Only destroy mutex if thread has stopped + // Only destroy mutexes if thread has stopped if (atomic_load(&ctx->state) == UDT_STATE_STOPPED) { pthread_mutex_destroy(&ctx->mutex); + pthread_mutex_destroy(&ctx->detection_stream_result_mutex); } else { log_warn("Skipping mutex destroy for %s - thread may still be running", ctx->stream_name); } @@ -595,6 +611,277 @@ static void stop_onvif_detection_thread(unified_detection_ctx_t *ctx) { log_info("[%s] ONVIF detection thread joined", ctx->stream_name); } +/* ============================================================ + * Secondary detection stream thread + * ============================================================ */ + +/* Sleep up to *delay_ms in 100 ms slices, checking the stop flag between + * slices, then exponentially back off *delay_ms (capped at MAX_RECONNECT_DELAY_MS). + * Mirrors the main UDT's reconnect cadence so a permanently bad detection_url + * doesn't hammer the upstream every 5 s. */ +static void detection_stream_backoff(unified_detection_ctx_t *ctx, int *delay_ms) { + int remaining = *delay_ms; + while (remaining > 0 && atomic_load(&ctx->detection_stream_thread_running)) { + int slice = remaining > 100 ? 100 : remaining; + av_usleep((unsigned)slice * 1000); + remaining -= slice; + } + *delay_ms *= 2; + if (*delay_ms > MAX_RECONNECT_DELAY_MS) *delay_ms = MAX_RECONNECT_DELAY_MS; +} + +/* Interrupt callback for the detection stream's blocking FFmpeg I/O. + * Without it, av_read_frame()/avformat_open_input() can block indefinitely on + * a network read (stimeout only covers RTSP, not HTTP/MJPEG), so a stop request + * or shutdown would wedge pthread_join() until a watchdog times out. Returns 1 + * to abort when the thread is asked to stop, the UDT is stopping, or the + * process is shutting down. */ +static int detection_stream_interrupt_cb(void *opaque) { + unified_detection_ctx_t *ctx = (unified_detection_ctx_t *)opaque; + if (!ctx) return 1; + if (!atomic_load(&ctx->detection_stream_thread_running) || + !atomic_load(&ctx->running) || + is_shutdown_initiated()) { + return 1; + } + return 0; +} + +static void *detection_stream_thread_func(void *arg) { + unified_detection_ctx_t *ctx = (unified_detection_ctx_t *)arg; + + log_info("[%s] Detection stream thread started (url=%s)", + ctx->stream_name, ctx->detection_stream_url); + + int reconnect_delay_ms = BASE_RECONNECT_DELAY_MS; + + while (atomic_load(&ctx->detection_stream_thread_running)) { + + /* ---- open stream ---- */ + /* Allocate the context up front so the interrupt callback is armed + * before avformat_open_input — making both the open and every later + * av_read_frame() abortable on stop/shutdown. */ + AVFormatContext *fmt_ctx = avformat_alloc_context(); + if (!fmt_ctx) { + log_warn("[%s] Detection stream thread: avformat_alloc_context failed", + ctx->stream_name); + detection_stream_backoff(ctx, &reconnect_delay_ms); + continue; + } + fmt_ctx->interrupt_callback.callback = detection_stream_interrupt_cb; + fmt_ctx->interrupt_callback.opaque = ctx; + + AVDictionary *opts = NULL; + av_dict_set(&opts, "rtsp_transport", "tcp", 0); + av_dict_set(&opts, "stimeout", "5000000", 0); + av_dict_set(&opts, "analyzeduration", "2000000", 0); + av_dict_set(&opts, "probesize", "2000000", 0); + /* Refuse file://, concat:, subfile: and other local-resource demuxers. + * detection_url is user-supplied; lock it to network protocols. */ + av_dict_set(&opts, "protocol_whitelist", + "udp,rtp,rtsp,tcp,https,tls,http", 0); + + int ret = avformat_open_input(&fmt_ctx, ctx->detection_stream_url, NULL, &opts); + av_dict_free(&opts); + + if (!atomic_load(&ctx->detection_stream_thread_running)) { + /* Stop was requested during the open call (potentially several + * seconds); exit silently rather than logging a misleading + * "cannot open ...: Success" warning. */ + if (fmt_ctx) avformat_close_input(&fmt_ctx); + break; + } + if (ret < 0) { + if (fmt_ctx) avformat_close_input(&fmt_ctx); + char errbuf[128]; + av_strerror(ret, errbuf, sizeof(errbuf)); + log_warn("[%s] Detection stream thread: cannot open %s: %s — retry in %d ms", + ctx->stream_name, ctx->detection_stream_url, errbuf, reconnect_delay_ms); + detection_stream_backoff(ctx, &reconnect_delay_ms); + continue; + } + + if (avformat_find_stream_info(fmt_ctx, NULL) < 0) { + log_warn("[%s] Detection stream thread: could not read stream info", ctx->stream_name); + avformat_close_input(&fmt_ctx); + detection_stream_backoff(ctx, &reconnect_delay_ms); + continue; + } + + int video_idx = -1; + for (unsigned int i = 0; i < fmt_ctx->nb_streams; i++) { + if (fmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + video_idx = (int)i; + break; + } + } + if (video_idx < 0) { + log_warn("[%s] Detection stream thread: no video stream in %s", + ctx->stream_name, ctx->detection_stream_url); + avformat_close_input(&fmt_ctx); + detection_stream_backoff(ctx, &reconnect_delay_ms); + continue; + } + + const AVCodec *decoder = + avcodec_find_decoder(fmt_ctx->streams[video_idx]->codecpar->codec_id); + AVCodecContext *dec_ctx = decoder ? avcodec_alloc_context3(decoder) : NULL; + if (!dec_ctx) { + log_warn("[%s] Detection stream thread: no decoder available", ctx->stream_name); + avformat_close_input(&fmt_ctx); + detection_stream_backoff(ctx, &reconnect_delay_ms); + continue; + } + avcodec_parameters_to_context(dec_ctx, fmt_ctx->streams[video_idx]->codecpar); + if (avcodec_open2(dec_ctx, decoder, NULL) < 0) { + log_warn("[%s] Detection stream thread: failed to open decoder", ctx->stream_name); + avcodec_free_context(&dec_ctx); + avformat_close_input(&fmt_ctx); + detection_stream_backoff(ctx, &reconnect_delay_ms); + continue; + } + + /* ---- read / produce loop ---- */ + time_t last_check = 0; + AVPacket *pkt = av_packet_alloc(); + AVFrame *frame = av_frame_alloc(); + bool stream_ok = (pkt != NULL && frame != NULL); + /* INTRA_ONLY is a codec-descriptor property, not a decoder capability + * (in the capabilities namespace bit 0 is DRAW_HORIZ_BAND). Use the + * descriptor so MJPEG and other all-keyframe codecs are recognized. */ + const AVCodecDescriptor *codec_desc = avcodec_descriptor_get(dec_ctx->codec_id); + bool is_intra_only = codec_desc && + (codec_desc->props & AV_CODEC_PROP_INTRA_ONLY) != 0; + + if (stream_ok) { + /* Allocations succeeded — only now signal to process_packet that it + * should suppress main-stream detection and consume from this thread. + * If pkt/frame failed we leave the flag clear so keyframe detection + * keeps running on the main stream. */ + atomic_store(&ctx->detection_stream_connected, 1); + reconnect_delay_ms = BASE_RECONNECT_DELAY_MS; + log_info("[%s] Detection stream thread connected", ctx->stream_name); + } else { + log_warn("[%s] Detection stream thread: failed to allocate packet/frame", + ctx->stream_name); + } + + while (atomic_load(&ctx->detection_stream_thread_running) && stream_ok) { + int rd = av_read_frame(fmt_ctx, pkt); + if (rd < 0) { + log_warn("[%s] Detection stream thread: read error, reconnecting", + ctx->stream_name); + stream_ok = false; + break; + } + + if (pkt->stream_index != video_idx) { + av_packet_unref(pkt); + continue; + } + + time_t now = time(NULL); + bool interval_elapsed = (now - last_check >= (time_t)ctx->detection_interval); + + /* Intra-only (MJPEG): discard frames within the interval at zero decode cost. + * Predictive codecs (H.264/H.265): must feed every packet to keep the + * reference chain intact; the interval gate moves into the frame-drain loop. */ + if (is_intra_only && !interval_elapsed) { + av_packet_unref(pkt); + continue; + } + + if (avcodec_send_packet(dec_ctx, pkt) < 0) { + av_packet_unref(pkt); + continue; + } + av_packet_unref(pkt); + + while (avcodec_receive_frame(dec_ctx, frame) == 0) { + now = time(NULL); + if (now - last_check >= (time_t)ctx->detection_interval) { + last_check = now; + + /* Pure detection — no DB writes, no state changes here. */ + detection_result_t result; + bool hit = detect_on_decoded_frame(ctx, frame, now, &result); + + if (hit) { + /* Write result to shared slot then raise the flag. + * process_packet() reads flag + slot and calls + * report_detections() + handle_recording_state(). */ + pthread_mutex_lock(&ctx->detection_stream_result_mutex); + ctx->detection_stream_pending = result; + pthread_mutex_unlock(&ctx->detection_stream_result_mutex); + atomic_store(&ctx->detection_stream_result, 1); + } + } + av_frame_unref(frame); + } + } + + if (pkt) av_packet_free(&pkt); + if (frame) av_frame_free(&frame); + avcodec_free_context(&dec_ctx); + avformat_close_input(&fmt_ctx); + + /* Clear connected flag and any pending result so the main loop falls + * back to main-stream detection immediately on the next keyframe. */ + atomic_store(&ctx->detection_stream_connected, 0); + atomic_store(&ctx->detection_stream_result, 0); + + if (atomic_load(&ctx->detection_stream_thread_running) && !stream_ok) { + log_info("[%s] Detection stream thread: reconnecting in %d ms", + ctx->stream_name, reconnect_delay_ms); + detection_stream_backoff(ctx, &reconnect_delay_ms); + } + } + + log_info("[%s] Detection stream thread exiting", ctx->stream_name); + return NULL; +} + +static int start_detection_stream_thread(unified_detection_ctx_t *ctx) { + atomic_store(&ctx->detection_stream_thread_running, 1); + atomic_store(&ctx->detection_stream_connected, 0); + atomic_store(&ctx->detection_stream_result, 0); + memset(&ctx->detection_stream_pending, 0, sizeof(ctx->detection_stream_pending)); + + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + + int ret = pthread_create(&ctx->detection_stream_thread, &attr, + detection_stream_thread_func, ctx); + pthread_attr_destroy(&attr); + + if (ret != 0) { + log_error("[%s] Failed to create detection stream thread: %s", + ctx->stream_name, strerror(ret)); + atomic_store(&ctx->detection_stream_thread_running, 0); + return -1; + } + + log_info("[%s] Detection stream thread created", ctx->stream_name); + return 0; +} + +static void stop_detection_stream_thread(unified_detection_ctx_t *ctx) { + int expected = 1; + if (!atomic_compare_exchange_strong(&ctx->detection_stream_thread_running, &expected, 0)) + return; + + log_info("[%s] Requesting detection stream thread to stop", ctx->stream_name); + + /* pthread_join blocks until the currently outstanding av_read_frame() + * (or avformat_open_input()) returns. stimeout=5000000 caps that at + * ~5 s for well-behaved demuxers; a half-open RTSP session may push the + * worst case higher. Acceptable because this path runs only during UDT + * teardown. The compare-exchange above guarantees only one caller joins. */ + pthread_join(ctx->detection_stream_thread, NULL); + log_info("[%s] Detection stream thread joined", ctx->stream_name); +} + /** * Start unified detection recording for a stream */ @@ -723,7 +1010,7 @@ int start_unified_detection_thread(const char *stream_name, const char *model_pa log_info("[%s] Built-in motion detection enabled (sensitivity=%.2f)", stream_name, sens); } - // Initialize mutex + // Initialize mutexes if (pthread_mutex_init(&ctx->mutex, NULL) != 0) { log_error("Failed to initialize mutex for unified detection context"); free(ctx); @@ -731,10 +1018,19 @@ int start_unified_detection_thread(const char *stream_name, const char *model_pa return -1; } + if (pthread_mutex_init(&ctx->detection_stream_result_mutex, NULL) != 0) { + log_error("Failed to initialize detection stream result mutex for %s", stream_name); + pthread_mutex_destroy(&ctx->mutex); + free(ctx); + pthread_mutex_unlock(&contexts_mutex); + return -1; + } + // Create circular buffer for pre-detection content ctx->packet_buffer = create_packet_buffer(stream_name, ctx->pre_buffer_seconds, BUFFER_MODE_MEMORY); if (!ctx->packet_buffer) { log_error("Failed to create pre-detection buffer for stream %s", stream_name); + pthread_mutex_destroy(&ctx->detection_stream_result_mutex); pthread_mutex_destroy(&ctx->mutex); free(ctx); pthread_mutex_unlock(&contexts_mutex); @@ -752,6 +1048,12 @@ int start_unified_detection_thread(const char *stream_name, const char *model_pa atomic_store(&ctx->onvif_motion_detected, 0); atomic_store(&ctx->onvif_motion_timestamp, 0LL); + // Initialize detection stream thread atomics and result slot + atomic_store(&ctx->detection_stream_thread_running, 0); + atomic_store(&ctx->detection_stream_connected, 0); + atomic_store(&ctx->detection_stream_result, 0); + memset(&ctx->detection_stream_pending, 0, sizeof(ctx->detection_stream_pending)); + // For ONVIF model: cache connection parameters and start the background // polling thread BEFORE the UDT starts reading packets. This ensures a // motion flag is available on the very first detection check. @@ -778,6 +1080,21 @@ int start_unified_detection_thread(const char *stream_name, const char *model_pa } } + // If a secondary detection URL is configured, cache it and start the detection stream thread. + if (config.detection_url[0] != '\0') { + safe_strcpy(ctx->detection_stream_url, config.detection_url, + sizeof(ctx->detection_stream_url), 0); + if (start_detection_stream_thread(ctx) != 0) { + log_warn("[%s] Detection stream thread could not be started; " + "falling back to keyframe-based detection on the main stream", + stream_name); + /* Non-fatal: the UDT will run detection on the main stream instead. */ + } else { + log_info("[%s] Detection stream thread started for url=%s", + stream_name, ctx->detection_stream_url); + } + } + // Store context in slot ctx->slot_idx = slot; detection_contexts[slot] = ctx; @@ -793,11 +1110,13 @@ int start_unified_detection_thread(const char *stream_name, const char *model_pa if (result != 0) { log_error("Failed to create unified detection thread for %s: %s", stream_name, strerror(result)); - /* Stop the ONVIF background thread before freeing ctx to avoid - * use-after-free: the thread was started above but the UDT that - * would normally join it never ran. */ + /* Stop background threads before freeing ctx to avoid use-after-free: + * the threads were started above but the UDT that would normally join + * them never ran. */ stop_onvif_detection_thread(ctx); + stop_detection_stream_thread(ctx); destroy_packet_buffer(ctx->packet_buffer); + pthread_mutex_destroy(&ctx->detection_stream_result_mutex); pthread_mutex_destroy(&ctx->mutex); free(ctx); detection_contexts[slot] = NULL; @@ -1288,7 +1607,6 @@ static void *unified_detection_thread_func(void *arg) { switch (state) { case UDT_STATE_INITIALIZING: log_info("[%s] State: INITIALIZING", stream_name); - // TODO: Load detection model state = UDT_STATE_CONNECTING; break; @@ -1482,6 +1800,9 @@ static void *unified_detection_thread_func(void *arg) { stop_onvif_detection_thread(ctx); } + // Stop the detection stream thread (no-op if never started). + stop_detection_stream_thread(ctx); + // Disconnect from stream to free FFmpeg decoder_ctx and input_ctx // This handles the case where the thread exits the loop while still connected // (e.g., during shutdown while in BUFFERING/RECORDING state) @@ -1509,6 +1830,7 @@ static void *unified_detection_thread_func(void *arg) { if (ctx->packet_buffer) { destroy_packet_buffer(ctx->packet_buffer); } + pthread_mutex_destroy(&ctx->detection_stream_result_mutex); pthread_mutex_destroy(&ctx->mutex); free(ctx); detection_contexts[i] = NULL; @@ -1677,12 +1999,19 @@ static int process_packet(unified_detection_ctx_t *ctx, AVPacket *pkt) { if (ctx->stream_is_live && is_keyframe) { should_buffer = true; } + } else if (ctx->stream_is_live) { + // Once the stream is live the replay-lag check is no longer meaningful: + // wall-clock and the camera's PTS clock can drift arbitrarily over + // long uptime, and gating buffering on that drift would freeze the + // pre-buffer with stale packets indefinitely (the !add path skips + // packet_buffer_add_packet, which is also where time-based eviction + // runs). Any real stall is handled by MAX_PACKET_TIMEOUT_SEC → + // reconnect, which re-anchors first_video_pts on the next session. + should_buffer = true; } else { - // Determine replay lag. - // If PTS is valid use PTS-vs-wallclock for accuracy. - // If PTS is AV_NOPTS_VALUE (common with some go2rtc outputs) - // fall back to pure wall-clock: treat the stream as still - // replaying until 2*pre_buffer_seconds have elapsed since connect. + // Pre-live: gate buffering until go2rtc's replay window closes. + // If PTS is valid use PTS-vs-wallclock for accuracy; otherwise fall + // back to a pure wall-clock warmup of 2 × pre_buffer_seconds. double wall_elapsed = difftime(now, ctx->stream_connect_time); double replay_lag; @@ -1691,32 +2020,23 @@ static int process_packet(unified_detection_ctx_t *ctx, AVPacket *pkt) { double pts_elapsed = (double)(pkt->pts - ctx->first_video_pts) * av_q2d(tb); replay_lag = wall_elapsed - pts_elapsed; } else { - // No PTS available — conservative wall-clock warmup double warmup = (double)(ctx->pre_buffer_seconds * 2); replay_lag = (wall_elapsed < warmup) ? (warmup - wall_elapsed) : 0.0; } if (replay_lag <= (double)ctx->pre_buffer_seconds) { - if (!ctx->stream_is_live) { - if (is_keyframe) { - // First live keyframe — flush stale data and start buffering - packet_buffer_clear(ctx->packet_buffer); - ctx->stream_is_live = true; - should_buffer = true; - log_info("[%s] go2rtc replay ended, pre-buffer active (lag=%.1fs)", - ctx->stream_name, replay_lag); - } - // else: waiting for first live keyframe — skip - } else { - should_buffer = true; - } - } else { - /* Rate-limit: only log on keyframes (~every 2 s) to avoid - * flooding the log at full frame-rate during replay. */ if (is_keyframe) { - log_debug("[%s] go2rtc replay in progress (lag=%.1fs), skipping pre-buffer", - ctx->stream_name, replay_lag); + // First live keyframe — flush stale replay data and go live. + packet_buffer_clear(ctx->packet_buffer); + ctx->stream_is_live = true; + should_buffer = true; + log_info("[%s] go2rtc replay ended, pre-buffer active (lag=%.1fs)", + ctx->stream_name, replay_lag); } + // else: waiting for first live keyframe — skip + } else if (is_keyframe) { + log_debug("[%s] go2rtc replay in progress (lag=%.1fs), skipping pre-buffer", + ctx->stream_name, replay_lag); } } } else if (!is_video && ctx->stream_is_live) { @@ -1731,6 +2051,39 @@ static int process_packet(unified_detection_ctx_t *ctx, AVPacket *pkt) { // Record stream metrics metrics_record_frame(ctx->stream_name, pkt->size, is_video); + // Detection stream path: runs on EVERY packet, no keyframe requirement. + // + // The background thread already handles its own rate-limiting and decoding. + // flush_prebuffer_to_recording() finds the first keyframe in the pre-buffer + // itself (see flush_packet_callback), so the current packet does not need + // to be a keyframe. Consuming the result on every packet rather than + // waiting for a keyframe reduces recording-start latency from up to one + // full GOP interval (2–4 s on typical cameras) to one packet interval (~40 ms). + // + // Consumed in every state, including POST_BUFFER, and *before* the + // post-buffer-expiry check below. Skipping consumption during POST_BUFFER + // left fresh hits sitting in the slot: the POST_BUFFER → RECORDING + // keep-alive branch in handle_recording_state was unreachable, and the + // stale slot then fired on the next packet after BUFFERING, triggering a + // phantom detection-less recording. + if (atomic_load(&ctx->detection_stream_connected)) { + bool detected = false; + if (atomic_exchange(&ctx->detection_stream_result, 0)) { + detection_result_t result; + pthread_mutex_lock(&ctx->detection_stream_result_mutex); + result = ctx->detection_stream_pending; + pthread_mutex_unlock(&ctx->detection_stream_result_mutex); + log_info("[%s] Detection stream result: %d detection(s)", + ctx->stream_name, result.count); + report_detections(ctx, &result, now); + detected = true; + } + handle_recording_state(ctx, detected, now); + // Re-read state: handle_recording_state may have transitioned + // BUFFERING → RECORDING or POST_BUFFER → RECORDING above. + current_state = (unified_detection_state_t)atomic_load(&ctx->state); + } + // If recording, write packet to MP4 if (current_state == UDT_STATE_RECORDING || current_state == UDT_STATE_POST_BUFFER) { if (ctx->mp4_writer && ctx->input_ctx) { @@ -1844,102 +2197,29 @@ static int process_packet(unified_detection_ctx_t *ctx, AVPacket *pkt) { current_state = (unified_detection_state_t)atomic_load(&ctx->state); } - // Run detection based on time interval (in seconds) - // We check on keyframes as a convenient trigger point, but the decision is time-based - // This ensures detection_interval is interpreted as seconds, not keyframe count - if (is_keyframe && current_state != UDT_STATE_POST_BUFFER) { + // Main-stream path: keyframe + time-gated. + // + // Decoding a non-keyframe packet in isolation is not possible without the + // preceding reference frames, so detection must wait for a keyframe. + // Only active when no detection stream is connected (fallback or unconfigured). + if (is_keyframe && current_state != UDT_STATE_POST_BUFFER && + !atomic_load(&ctx->detection_stream_connected)) { time_t time_since_last_check = now - (time_t)atomic_load(&ctx->last_detection_check_time); - // Log periodically to show detection is running if (time_since_last_check > 0 && (atomic_fetch_add(&ctx->log_counter, 1) % 10) == 0) { log_debug("[%s] Time since last detection check: %ld/%d seconds, model=%s, state=%d", ctx->stream_name, (long)time_since_last_check, ctx->detection_interval, ctx->model_path, current_state); } - // Run detection if enough time has passed (detection_interval is in seconds) if (time_since_last_check >= ctx->detection_interval) { atomic_store(&ctx->last_detection_check_time, (long long)now); - log_info("[%s] Running detection (interval=%ds, elapsed=%lds, model=%s)", - ctx->stream_name, ctx->detection_interval, (long)time_since_last_check, ctx->model_path); - - // Decode frame and run detection - bool detection_triggered = run_detection_on_frame(ctx, pkt); - - // If detection triggered - if (detection_triggered) { - atomic_store(&ctx->last_detection_time, (long long)now); - - pthread_mutex_lock(&ctx->mutex); - ctx->total_detections++; - pthread_mutex_unlock(&ctx->mutex); - - // In annotation_only mode, we don't manage recording state - just store detections - // The continuous recording system handles the actual MP4 files - if (!ctx->annotation_only) { - // If not already recording, start recording - if (current_state == UDT_STATE_BUFFERING) { - log_info("[%s] Detection triggered, starting recording", ctx->stream_name); - - // Start recording first, then flush pre-buffer - if (udt_start_recording(ctx) == 0) { - // Flush pre-buffer and correct DB start_time - int pre_dur = 0; - int pre_cnt = 0; size_t pre_mem = 0; - if (ctx->packet_buffer) - packet_buffer_get_stats(ctx->packet_buffer, &pre_cnt, &pre_mem, &pre_dur); - - flush_prebuffer_to_recording(ctx); - atomic_store(&ctx->state, UDT_STATE_RECORDING); - - // Correct start_time in DB and writer to the actual first-packet time - if (!ctx->mp4_writer->start_time_corrected && pre_dur > 0 && - ctx->current_recording_id > 0) { - // Clamp pre_dur to the configured pre_buffer window. - // go2rtc may deliver a ring-buffer of 200+ seconds; using - // the raw value would push start_time so far back that - // elapsed > max_duration immediately, stopping the recording. - int clamped_pre = pre_dur > ctx->pre_buffer_seconds - ? ctx->pre_buffer_seconds : pre_dur; - time_t corrected = now - (time_t)clamped_pre; - ctx->mp4_writer->creation_time = corrected; - ctx->mp4_writer->start_time_corrected = true; - update_recording_start_time(ctx->current_recording_id, corrected); - log_info("[%s] Corrected recording start_time by -%ds (pre-buffer, clamped from %ds)", - ctx->stream_name, clamped_pre, pre_dur); - } - - // Link any recent detections (that triggered this recording) to the new recording_id - // Look back up to detection_interval + 2 seconds to catch the triggering detection - time_t lookback = now - (ctx->detection_interval > 0 ? ctx->detection_interval + 2 : 7); - int updated = update_detections_recording_id(ctx->stream_name, - ctx->current_recording_id, - lookback); - if (updated > 0) { - log_debug("[%s] Linked %d recent detections to recording ID %lu", - ctx->stream_name, updated, (unsigned long)ctx->current_recording_id); - } - } - } - // If in post-buffer, go back to recording - else if (current_state == UDT_STATE_POST_BUFFER) { - log_info("[%s] Detection during post-buffer, continuing recording", ctx->stream_name); - atomic_store(&ctx->state, UDT_STATE_RECORDING); - } - } - } - // No detection - check if we should enter post-buffer (only in detection-recording mode) - else if (!ctx->annotation_only && current_state == UDT_STATE_RECORDING) { - // Check if enough time has passed since last detection - if (now - (time_t)atomic_load(&ctx->last_detection_time) > DETECTION_GRACE_PERIOD_SEC) { // grace period before post-buffer - log_info("[%s] No detection, entering post-buffer (%d seconds)", - ctx->stream_name, ctx->post_buffer_seconds); - atomic_store(&ctx->post_buffer_end_time, (long long)(now + ctx->post_buffer_seconds)); - atomic_store(&ctx->state, UDT_STATE_POST_BUFFER); - } - } + ctx->stream_name, ctx->detection_interval, + (long)time_since_last_check, ctx->model_path); + bool detected = run_detection_on_frame(ctx, pkt); + handle_recording_state(ctx, detected, now); } } @@ -1987,6 +2267,7 @@ static int udt_start_recording(unified_detection_ctx_t *ctx) { log_error("[%s] Failed to create MP4 writer", ctx->stream_name); return -1; } + ctx->mp4_writer->pre_buffer_seconds = ctx->pre_buffer_seconds; // Configure audio recording based on stream settings if (ctx->record_audio && ctx->audio_stream_idx >= 0) { @@ -2216,437 +2497,344 @@ static int flush_prebuffer_to_recording(unified_detection_ctx_t *ctx) { static bool run_detection_on_frame(unified_detection_ctx_t *ctx, AVPacket *pkt) { if (!ctx) return false; - detection_result_t result; - memset(&result, 0, sizeof(detection_result_t)); + time_t now = time(NULL); - // Check if this is API-based detection + /* API detection: try the go2rtc snapshot shortcut first. + * The snapshot path handles zone filtering, DB storage, and MQTT + * internally so we only need to check the threshold on its result. + * If the snapshot is unavailable we fall through to the shared + * frame-decode path below. */ if (is_api_detection(ctx->model_path)) { - // API detection - try go2rtc snapshot first (more efficient, no frame decoding needed) log_debug("[%s] Running API detection via snapshot", ctx->stream_name); - // Determine recording_id to link detections to uint64_t rec_id = 0; if (ctx->annotation_only) { - // In annotation_only mode, link detections to the continuous recording rec_id = get_current_recording_id_for_stream(ctx->stream_name); } else if (ctx->current_recording_id > 0) { - // For detection recordings, link to the current detection recording rec_id = ctx->current_recording_id; } - // The model_path contains either "api-detection" or an HTTP URL - // detect_objects_api_snapshot handles the "api-detection" special case - // by looking up g_config.api_detection_url + detection_result_t result; + memset(&result, 0, sizeof(result)); int detect_ret = detect_objects_api_snapshot(ctx->model_path, ctx->stream_name, &result, ctx->detection_threshold, rec_id); - if (detect_ret == DETECT_SNAPSHOT_UNAVAILABLE) { - // go2rtc snapshot failed - fall back to local frame decoding - log_info("[%s] go2rtc snapshot unavailable, falling back to local frame decode", ctx->stream_name); - - // We need a packet and decoder to fall back - if (!pkt || !ctx->decoder_ctx) { - log_debug("[%s] Cannot fall back: no packet or decoder available", ctx->stream_name); - return false; - } - - // Decode the packet to get a frame - int ret = avcodec_send_packet(ctx->decoder_ctx, pkt); - if (ret < 0) { - log_debug("[%s] Fallback decode failed: avcodec_send_packet error %d", ctx->stream_name, ret); - return false; - } - - AVFrame *frame = av_frame_alloc(); - if (!frame) { - log_debug("[%s] Fallback decode failed: could not allocate frame", ctx->stream_name); - return false; - } - - ret = avcodec_receive_frame(ctx->decoder_ctx, frame); - if (ret < 0) { - av_frame_free(&frame); - log_debug("[%s] Fallback decode failed: avcodec_receive_frame error %d", ctx->stream_name, ret); - return false; - } - - // Convert frame to RGB for API detection - int width = frame->width; - int height = frame->height; - int channels = 3; // RGB - - struct SwsContext *sws_ctx = sws_getContext( - width, height, frame->format, - width, height, AV_PIX_FMT_RGB24, - SWS_BILINEAR, NULL, NULL, NULL); - - if (!sws_ctx) { - log_error("[%s] Fallback: failed to create sws context", ctx->stream_name); - av_frame_free(&frame); + if (detect_ret != DETECT_SNAPSHOT_UNAVAILABLE) { + if (detect_ret != 0) { + log_warn("[%s] API detection failed with error %d", ctx->stream_name, detect_ret); return false; } - - size_t rgb_buffer_size = (size_t)width * height * channels; - uint8_t *rgb_buffer = malloc(rgb_buffer_size); - if (!rgb_buffer) { - log_error("[%s] Fallback: failed to allocate RGB buffer", ctx->stream_name); - sws_freeContext(sws_ctx); - av_frame_free(&frame); - return false; + /* Snapshot succeeded — snapshot handles DB storage internally. + * Call report_detections for stats only (no-op for DB on API). */ + report_detections(ctx, &result, now); + for (int i = 0; i < result.count; i++) { + if (result.detections[i].confidence >= ctx->detection_threshold) { + log_info("[%s] API Detection: %s (%.1f%%) at [%.2f, %.2f, %.2f, %.2f]", + ctx->stream_name, + result.detections[i].label, + result.detections[i].confidence * 100.0f, + result.detections[i].x, result.detections[i].y, + result.detections[i].width, result.detections[i].height); + return true; + } } + return false; + } - uint8_t *rgb_data[4] = {rgb_buffer, NULL, NULL, NULL}; - int rgb_linesize[4] = {width * channels, 0, 0, 0}; - - sws_scale(sws_ctx, (const uint8_t * const *)frame->data, frame->linesize, - 0, height, rgb_data, rgb_linesize); - - sws_freeContext(sws_ctx); - av_frame_free(&frame); - - // Get the actual API URL - const char *actual_api_url = get_actual_api_url(ctx->stream_name, ctx->model_path); - if (actual_api_url == NULL) { - free(rgb_buffer); - return false; - } + log_info("[%s] go2rtc snapshot unavailable, falling back to frame decode", + ctx->stream_name); + /* Fall through to frame-decode path below. */ + } - log_info("[%s] Fallback: sending %dx%d frame to API detection", ctx->stream_name, width, height); + /* ONVIF: event-based — no frame needed, just read the async-thread flag. + * + * detect_motion_onvif() is NOT called here. A dedicated background thread + * (start_onvif_detection_thread) polls the camera and writes the result + * into ctx->onvif_motion_detected, keeping av_read_frame() unblocked. */ + if (is_onvif_detection_model(ctx->model_path)) { + bool triggered = (atomic_load(&ctx->onvif_motion_detected) != 0); + if (triggered) { + pthread_mutex_lock(&ctx->mutex); + ctx->total_detections++; + pthread_mutex_unlock(&ctx->mutex); + log_info("[%s] ONVIF motion detected (async thread, ts=%lld)", + ctx->stream_name, + (long long)atomic_load(&ctx->onvif_motion_timestamp)); + } + return triggered; + } - // Call API detection with decoded frame (rec_id already computed above) - detect_ret = detect_objects_api(actual_api_url, rgb_buffer, width, height, channels, - &result, ctx->stream_name, ctx->detection_threshold, rec_id); + /* All other models (motion, SOD/TFLite, API snapshot fallback): + * decode packet → detect_on_decoded_frame → report_detections. */ + if (!pkt || !ctx->decoder_ctx) return false; - free(rgb_buffer); + if (avcodec_send_packet(ctx->decoder_ctx, pkt) < 0) return false; - if (detect_ret != 0) { - log_warn("[%s] Fallback API detection failed with error %d", ctx->stream_name, detect_ret); - return false; - } + AVFrame *frame = av_frame_alloc(); + if (!frame) return false; - log_info("[%s] Fallback API detection successful: %d objects detected", ctx->stream_name, result.count); - } else if (detect_ret != 0) { - log_warn("[%s] API detection failed with error %d", ctx->stream_name, detect_ret); - return false; - } + if (avcodec_receive_frame(ctx->decoder_ctx, frame) < 0) { + av_frame_free(&frame); + return false; + } - // Note: detect_objects_api_snapshot already handles: - // - Filtering by zones - // - Storing in database - // - MQTT publishing - - // Check if any detections meet the threshold and trigger recording - bool detection_triggered = false; - for (int i = 0; i < result.count; i++) { - if (result.detections[i].confidence >= ctx->detection_threshold) { - detection_triggered = true; - log_info("[%s] API Detection: %s (%.1f%%) at [%.2f, %.2f, %.2f, %.2f]", - ctx->stream_name, - result.detections[i].label, - result.detections[i].confidence * 100.0f, - result.detections[i].x, - result.detections[i].y, - result.detections[i].width, - result.detections[i].height); - } - } + detection_result_t result2; + bool hit = detect_on_decoded_frame(ctx, frame, now, &result2); + av_frame_free(&frame); - return detection_triggered; - } + if (hit) report_detections(ctx, &result2, now); + return hit; +} +/* ============================================================ + * Detection layer — pure inference, no side effects + * ============================================================ */ - // Built-in motion detection - requires frame decoding but no external model file - if (is_motion_detection_model(ctx->model_path)) { - if (!pkt || !ctx->decoder_ctx) return false; +/** + * Run the configured model on a pre-decoded frame and return the surviving + * detections via *result. Returns true when result->count > 0. + * + * Pure computation: zone + object + threshold filters are applied, but no DB + * writes, no recording-state changes, and no MQTT publishing happen here. + * Callers pass the result to report_detections() then handle_recording_state(). + * + * Exception: API backends (detect_objects_api / detect_objects_api_snapshot) + * store results in the DB internally — this is their pre-existing design. + * report_detections() is therefore a no-op for API models. + * + * Handles: API (frame path), motion detection, SOD/TFLite embedded models. + * Does NOT handle ONVIF (event-based, no frame needed). + */ +/* Convert a decoded frame to a freshly malloc'd packed RGB24 buffer at the + * frame's own resolution (caller frees). Returns NULL on failure. */ +static uint8_t *frame_to_rgb24(const AVFrame *frame) { + struct SwsContext *sws_ctx = sws_getContext( + frame->width, frame->height, frame->format, + frame->width, frame->height, AV_PIX_FMT_RGB24, + SWS_BILINEAR, NULL, NULL, NULL); + if (!sws_ctx) return NULL; - // Decode the packet to get a frame - int ret = avcodec_send_packet(ctx->decoder_ctx, pkt); - if (ret < 0) { - return false; - } + uint8_t *rgb_buf = malloc((size_t)frame->width * frame->height * 3); + if (!rgb_buf) { sws_freeContext(sws_ctx); return NULL; } - AVFrame *motion_frame = av_frame_alloc(); - if (!motion_frame) { - return false; - } + uint8_t *rgb_data[4] = {rgb_buf, NULL, NULL, NULL}; + int rgb_linesize[4] = {frame->width * 3, 0, 0, 0}; + sws_scale(sws_ctx, (const uint8_t * const *)frame->data, frame->linesize, + 0, frame->height, rgb_data, rgb_linesize); + sws_freeContext(sws_ctx); + return rgb_buf; +} - ret = avcodec_receive_frame(ctx->decoder_ctx, motion_frame); - if (ret < 0) { - av_frame_free(&motion_frame); - return false; - } +static bool detect_on_decoded_frame(unified_detection_ctx_t *ctx, + AVFrame *frame, time_t now, + detection_result_t *result) { + if (!ctx || !frame || !result) return false; - // Convert frame to RGB for motion detection - int mot_width = motion_frame->width; - int mot_height = motion_frame->height; - int mot_channels = 3; // RGB + memset(result, 0, sizeof(*result)); - struct SwsContext *mot_sws_ctx = sws_getContext( - mot_width, mot_height, motion_frame->format, - mot_width, mot_height, AV_PIX_FMT_RGB24, - SWS_BILINEAR, NULL, NULL, NULL); + int width = frame->width; + int height = frame->height; - if (!mot_sws_ctx) { - log_error("[%s] Failed to create sws context for motion detection", ctx->stream_name); - av_frame_free(&motion_frame); - return false; - } + /* ---- model dispatch ---- */ - size_t mot_buffer_size = (size_t)mot_width * mot_height * mot_channels; - uint8_t *mot_rgb_buffer = malloc(mot_buffer_size); - if (!mot_rgb_buffer) { - log_error("[%s] Failed to allocate RGB buffer for motion detection", ctx->stream_name); - sws_freeContext(mot_sws_ctx); - av_frame_free(&motion_frame); + if (is_api_detection(ctx->model_path)) { + /* API backends store in DB internally using rec_id for recording linking. */ + uint64_t rec_id = ctx->annotation_only + ? get_current_recording_id_for_stream(ctx->stream_name) + : ctx->current_recording_id; + + uint8_t *rgb_buf = frame_to_rgb24(frame); + if (!rgb_buf) return false; + + const char *api_url = get_actual_api_url(ctx->stream_name, ctx->model_path); + int ret = api_url + ? detect_objects_api(api_url, rgb_buf, width, height, 3, + result, ctx->stream_name, ctx->detection_threshold, rec_id) + : -1; + free(rgb_buf); + + if (ret != 0) { + log_warn("[%s] API detection (frame path) failed with error %d", + ctx->stream_name, ret); return false; } - uint8_t *mot_rgb_data[4] = {mot_rgb_buffer, NULL, NULL, NULL}; - int mot_rgb_linesize[4] = {mot_width * mot_channels, 0, 0, 0}; - - sws_scale(mot_sws_ctx, (const uint8_t * const *)motion_frame->data, motion_frame->linesize, - 0, mot_height, mot_rgb_data, mot_rgb_linesize); - - sws_freeContext(mot_sws_ctx); - av_frame_free(&motion_frame); - - // Run built-in motion detection - time_t mot_frame_time = time(NULL); - int mot_ret = detect_motion(ctx->stream_name, mot_rgb_buffer, mot_width, mot_height, - mot_channels, mot_frame_time, &result); + } else if (is_motion_detection_model(ctx->model_path)) { + uint8_t *rgb_buf = frame_to_rgb24(frame); + if (!rgb_buf) return false; - free(mot_rgb_buffer); + int ret = detect_motion(ctx->stream_name, rgb_buf, width, height, 3, now, result); + free(rgb_buf); - if (mot_ret != 0) { - log_warn("[%s] Motion detection failed with error %d", ctx->stream_name, mot_ret); + if (ret != 0) { + log_warn("[%s] Motion detection failed with error %d", ctx->stream_name, ret); return false; } - // Apply zone filtering to motion detections (consistent with ONVIF path) - if (result.count > 0) { - int zf_ret = filter_detections_by_zones(ctx->stream_name, &result); - if (zf_ret != 0) { - log_warn("[%s] Zone filtering failed for motion detections, keeping all", - ctx->stream_name); + } else { + /* Embedded model (SOD / TFLite / etc.). */ + if (!ctx->model) { + if (ctx->model_path[0] == '\0') return false; + ctx->model = load_detection_model(ctx->model_path, ctx->detection_threshold); + if (!ctx->model) { + log_warn("[%s] Failed to load detection model: %s", + ctx->stream_name, ctx->model_path); + return false; } + log_info("[%s] Loaded detection model: %s", ctx->stream_name, ctx->model_path); } - // Filter out detections below the threshold so they are not stored - // or displayed on the overlay. Keep only those that meet the - // configured detection_threshold. - bool mot_triggered = false; - { - int kept = 0; - for (int i = 0; i < result.count; i++) { - if (result.detections[i].confidence >= ctx->detection_threshold) { - mot_triggered = true; - log_info("[%s] Motion detected: %s (%.1f%%)", - ctx->stream_name, - result.detections[i].label, - result.detections[i].confidence * 100.0f); - if (kept != i) { - result.detections[kept] = result.detections[i]; - } - kept++; - } else { - log_debug("[%s] Motion below threshold: %s (%.1f%% < %.1f%%)", - ctx->stream_name, - result.detections[i].label, - result.detections[i].confidence * 100.0f, - ctx->detection_threshold * 100.0f); - } - } - result.count = kept; + uint8_t *rgb_buf = frame_to_rgb24(frame); + if (!rgb_buf) { + log_error("[%s] Failed to convert frame to RGB24", ctx->stream_name); + return false; } - // Store detections in database if any passed the threshold - if (result.count > 0) { - time_t now = time(NULL); - uint64_t rec_id = 0; - if (ctx->annotation_only) { - rec_id = get_current_recording_id_for_stream(ctx->stream_name); - } else if (ctx->current_recording_id > 0) { - rec_id = ctx->current_recording_id; - } - if (store_detections_in_db(ctx->stream_name, &result, now, rec_id) != 0) { - log_warn("[%s] Failed to store motion detections in database", ctx->stream_name); - } + int ret = detect_objects(ctx->model, rgb_buf, width, height, 3, result); + free(rgb_buf); - // Keep MQTT/Home Assistant motion topics in sync with built-in - // motion detections. Other detection backends publish here via - // their own pipelines; built-in motion must do the same after - // threshold and zone filtering. - mqtt_publish_detection(ctx->stream_name, &result, now); - mqtt_set_motion_state(ctx->stream_name, &result); - - pthread_mutex_lock(&ctx->mutex); - ctx->total_detections += result.count; - pthread_mutex_unlock(&ctx->mutex); + if (ret != 0) { + log_warn("[%s] Embedded model detection failed with error %d", + ctx->stream_name, ret); + return false; } - - return mot_triggered; } - // ONVIF event-based detection — non-blocking atomic flag read - // ----------------------------------------------------------------------- - // detect_motion_onvif() is NOT called here anymore. A dedicated - // background thread (started alongside the UDT, see - // start_onvif_detection_thread) polls the camera continuously and writes - // its result into ctx->onvif_motion_detected. - // - // This keeps process_packet() / av_read_frame() completely unblocked: - // • No CURL/SOAP round-trip in the UDT main loop. - // • No PTS gaps in the MP4 caused by ONVIF blocking. - // • No write i/o timeout on the go2rtc consumer connection. - // - // The ONVIF thread manages the flag value: - // 1 while the camera reports motion events; 0 when idle or on error. - // We read without clearing — the thread updates the flag each poll cycle. - // - // SOD and API detection paths are fully unaffected by this change. - // ----------------------------------------------------------------------- - if (is_onvif_detection_model(ctx->model_path)) { - bool onvif_triggered = (atomic_load(&ctx->onvif_motion_detected) != 0); + /* ---- post-processing: common to all model types ---- */ - if (onvif_triggered) { - pthread_mutex_lock(&ctx->mutex); - ctx->total_detections++; - pthread_mutex_unlock(&ctx->mutex); - log_info("[%s] ONVIF motion detected (async thread, ts=%lld)", - ctx->stream_name, - (long long)atomic_load(&ctx->onvif_motion_timestamp)); - } - return onvif_triggered; + if (result->count == 0) return false; - } - - // Embedded model detection - requires frame decoding - if (!pkt || !ctx->decoder_ctx) return false; + filter_detections_by_zones(ctx->stream_name, result); + filter_detections_by_stream_objects(ctx->stream_name, result); - // Check if we have a detection model loaded - if (!ctx->model) { - // Try to load the model if we have a path - if (ctx->model_path[0] != '\0') { - ctx->model = load_detection_model(ctx->model_path, ctx->detection_threshold); - if (!ctx->model) { - log_warn("[%s] Failed to load detection model: %s", ctx->stream_name, ctx->model_path); - return false; - } - log_info("[%s] Loaded detection model: %s", ctx->stream_name, ctx->model_path); + /* Compact in-place: discard detections below threshold. */ + int kept = 0; + for (int i = 0; i < result->count; i++) { + if (result->detections[i].confidence >= ctx->detection_threshold) { + log_info("[%s] Detection: %s (%.1f%%) at [%.2f, %.2f, %.2f, %.2f]", + ctx->stream_name, + result->detections[i].label, + result->detections[i].confidence * 100.0f, + result->detections[i].x, result->detections[i].y, + result->detections[i].width, result->detections[i].height); + if (kept != i) result->detections[kept] = result->detections[i]; + kept++; } else { - return false; + log_debug("[%s] Detection below threshold: %s (%.1f%% < %.1f%%)", + ctx->stream_name, + result->detections[i].label, + result->detections[i].confidence * 100.0f, + ctx->detection_threshold * 100.0f); } } + result->count = kept; + return result->count > 0; +} - // Decode the packet to get a frame - int ret = avcodec_send_packet(ctx->decoder_ctx, pkt); - if (ret < 0) { - return false; - } +/* ============================================================ + * Action layer + * ============================================================ */ - AVFrame *frame = av_frame_alloc(); - if (!frame) { - return false; - } +/** + * Store detection results in the database and update stream statistics. + * + * No-op for API models: detect_objects_api / detect_objects_api_snapshot + * already write to the DB internally. For all other models (motion, SOD, + * TFLite) this is the single place where DB writes and stats happen. + */ +static void report_detections(unified_detection_ctx_t *ctx, + const detection_result_t *result, time_t now) { + if (!ctx || !result || result->count == 0) return; - ret = avcodec_receive_frame(ctx->decoder_ctx, frame); - if (ret < 0) { - av_frame_free(&frame); - return false; - } + if (!is_api_detection(ctx->model_path)) { + uint64_t rec_id = ctx->annotation_only + ? get_current_recording_id_for_stream(ctx->stream_name) + : ctx->current_recording_id; - // Convert frame to RGB for detection - int width = frame->width; - int height = frame->height; - int channels = 3; // RGB - - // Create software scaler for conversion - struct SwsContext *sws_ctx = sws_getContext( - width, height, frame->format, - width, height, AV_PIX_FMT_RGB24, - SWS_BILINEAR, NULL, NULL, NULL); + if (store_detections_in_db(ctx->stream_name, result, now, rec_id) != 0) + log_warn("[%s] Failed to store detections in database", ctx->stream_name); - if (!sws_ctx) { - log_error("[%s] Failed to create sws context", ctx->stream_name); - av_frame_free(&frame); - return false; + // Keep MQTT/Home Assistant topics in sync for the local detection + // backends (motion, SOD, TFLite). API backends publish from inside + // detect_objects_api / *_snapshot, so they are excluded here. + mqtt_publish_detection(ctx->stream_name, result, now); + mqtt_set_motion_state(ctx->stream_name, result); } - // Allocate RGB buffer - size_t rgb_buffer_size = (size_t)width * height * channels; - uint8_t *rgb_buffer = malloc(rgb_buffer_size); - if (!rgb_buffer) { - log_error("[%s] Failed to allocate RGB buffer", ctx->stream_name); - sws_freeContext(sws_ctx); - av_frame_free(&frame); - return false; - } + pthread_mutex_lock(&ctx->mutex); + ctx->total_detections += (uint64_t)result->count; + pthread_mutex_unlock(&ctx->mutex); +} - // Convert frame to RGB - uint8_t *rgb_data[4] = {rgb_buffer, NULL, NULL, NULL}; - int rgb_linesize[4] = {width * channels, 0, 0, 0}; +/** + * Drive the recording state machine based on whether detection fired this + * interval. This is the single place where BUFFERING→RECORDING, + * RECORDING→POST_BUFFER, and POST_BUFFER→RECORDING transitions happen for + * the frame-based detection paths (main stream and detection stream). + * + * ONVIF and cross-stream motion triggers follow the external_motion_trigger + * path in process_packet and do not call this function. + */ +static void handle_recording_state(unified_detection_ctx_t *ctx, + bool detected, time_t now) { + if (!ctx) return; - sws_scale(sws_ctx, (const uint8_t * const *)frame->data, frame->linesize, - 0, height, rgb_data, rgb_linesize); + unified_detection_state_t current_state = + (unified_detection_state_t)atomic_load(&ctx->state); - sws_freeContext(sws_ctx); - av_frame_free(&frame); + if (detected) { + atomic_store(&ctx->last_detection_time, (long long)now); - // Run detection - int detect_ret = detect_objects(ctx->model, rgb_buffer, width, height, channels, &result); + if (ctx->annotation_only) return; - free(rgb_buffer); + if (current_state == UDT_STATE_BUFFERING) { + log_info("[%s] Detection triggered, starting recording", ctx->stream_name); - if (detect_ret != 0) { - log_warn("[%s] Detection failed with error %d", ctx->stream_name, detect_ret); - return false; - } + if (udt_start_recording(ctx) == 0) { + int pre_dur = 0, pre_cnt = 0; + size_t pre_mem = 0; + if (ctx->packet_buffer) + packet_buffer_get_stats(ctx->packet_buffer, &pre_cnt, &pre_mem, &pre_dur); - // Check if any detections meet the threshold - bool detection_triggered = false; - for (int i = 0; i < result.count; i++) { - if (result.detections[i].confidence >= ctx->detection_threshold) { - detection_triggered = true; - log_info("[%s] Detection: %s (%.1f%%) at [%.2f, %.2f, %.2f, %.2f]", - ctx->stream_name, - result.detections[i].label, - result.detections[i].confidence * 100.0f, - result.detections[i].x, - result.detections[i].y, - result.detections[i].width, - result.detections[i].height); - } - } + flush_prebuffer_to_recording(ctx); + atomic_store(&ctx->state, UDT_STATE_RECORDING); - // Store detections in database if any were found - if (result.count > 0) { - time_t now = time(NULL); + if (!ctx->mp4_writer->start_time_corrected && pre_dur > 0 && + ctx->current_recording_id > 0) { + int clamped_pre = pre_dur > ctx->pre_buffer_seconds + ? ctx->pre_buffer_seconds : pre_dur; + time_t corrected = now - (time_t)clamped_pre; + ctx->mp4_writer->creation_time = corrected; + ctx->mp4_writer->start_time_corrected = true; + update_recording_start_time(ctx->current_recording_id, corrected); + log_info("[%s] Corrected recording start_time by -%ds (pre-buffer, clamped from %ds)", + ctx->stream_name, clamped_pre, pre_dur); + } - // Link detections to the current recording - uint64_t rec_id = 0; - if (ctx->annotation_only) { - // In annotation_only mode, link detections to the continuous recording - rec_id = get_current_recording_id_for_stream(ctx->stream_name); - if (rec_id > 0) { - log_debug("[%s] Annotation mode: linking detections to recording ID %lu", - ctx->stream_name, (unsigned long)rec_id); - } else { - log_debug("[%s] Annotation mode: no active recording to link detections to", - ctx->stream_name); + time_t lookback = now - (ctx->detection_interval > 0 + ? ctx->detection_interval + 2 : 7); + int updated = update_detections_recording_id( + ctx->stream_name, ctx->current_recording_id, lookback); + if (updated > 0) + log_debug("[%s] Linked %d recent detections to recording ID %lu", + ctx->stream_name, updated, + (unsigned long)ctx->current_recording_id); } - } else if (ctx->current_recording_id > 0) { - // For detection recordings, link to the current detection recording - rec_id = ctx->current_recording_id; - log_debug("[%s] Detection mode: linking detections to recording ID %lu", - ctx->stream_name, (unsigned long)rec_id); - } - if (store_detections_in_db(ctx->stream_name, &result, now, rec_id) != 0) { - log_warn("[%s] Failed to store detections in database", ctx->stream_name); + } else if (current_state == UDT_STATE_POST_BUFFER) { + log_info("[%s] Detection during post-buffer, continuing recording", + ctx->stream_name); + atomic_store(&ctx->state, UDT_STATE_RECORDING); + } + /* UDT_STATE_RECORDING: last_detection_time already refreshed above. */ + + } else if (!ctx->annotation_only && current_state == UDT_STATE_RECORDING) { + time_t since_last = now - (time_t)atomic_load(&ctx->last_detection_time); + if (since_last > DETECTION_GRACE_PERIOD_SEC) { + log_info("[%s] No detection, entering post-buffer (%d seconds)", + ctx->stream_name, ctx->post_buffer_seconds); + atomic_store(&ctx->post_buffer_end_time, + (long long)(now + ctx->post_buffer_seconds)); + atomic_store(&ctx->state, UDT_STATE_POST_BUFFER); } - pthread_mutex_lock(&ctx->mutex); - ctx->total_detections += result.count; - pthread_mutex_unlock(&ctx->mutex); } - - return detection_triggered; } From e9214eaea8cd6e76b671dece65be4cd561a08951 Mon Sep 17 00:00:00 2001 From: Piotr Ziecik Date: Fri, 19 Jun 2026 10:44:16 +0200 Subject: [PATCH 04/13] feat(web): detection URL field in stream config + i18n Add the optional "Detection Stream URL" field to the stream config modal (create/edit/clone round-trip via the detection_url payload key) and the streamsConfig.detectionUrl* strings to all 17 locales. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../components/preact/StreamConfigModal.jsx | 18 ++++++++++++++++++ web/js/components/preact/StreamsView.jsx | 19 ++++++++++++++----- web/public/locales/ar.json | 3 +++ web/public/locales/de.json | 3 +++ web/public/locales/en.json | 3 +++ web/public/locales/es.json | 3 +++ web/public/locales/fr.json | 3 +++ web/public/locales/it.json | 3 +++ web/public/locales/ja.json | 3 +++ web/public/locales/ko.json | 3 +++ web/public/locales/nl.json | 3 +++ web/public/locales/pl.json | 3 +++ web/public/locales/pt-BR.json | 3 +++ web/public/locales/pt-PT.json | 3 +++ web/public/locales/ru.json | 3 +++ web/public/locales/tr.json | 3 +++ web/public/locales/uk.json | 3 +++ web/public/locales/zh-TW.json | 3 +++ web/public/locales/zh.json | 3 +++ 19 files changed, 83 insertions(+), 5 deletions(-) diff --git a/web/js/components/preact/StreamConfigModal.jsx b/web/js/components/preact/StreamConfigModal.jsx index acdd7862..997a0d7e 100644 --- a/web/js/components/preact/StreamConfigModal.jsx +++ b/web/js/components/preact/StreamConfigModal.jsx @@ -973,6 +973,24 @@ export function StreamConfigModal({ )} +
+ + +

+ {t('streamsConfig.detectionUrlHelp')} +

+
+
+
+ +
+ + {t('settings.detectionGracePeriodHelp')} +
+
); diff --git a/web/public/locales/ar.json b/web/public/locales/ar.json index 35a8b7d3..7fd4b7b4 100644 --- a/web/public/locales/ar.json +++ b/web/public/locales/ar.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "محرك الاستدلال المستخدم بواسطة API الاكتشاف", "settings.defaultDetectionThreshold": "عتبة الاكتشاف الافتراضية", "settings.defaultDetectionThresholdHelp": "عتبة الثقة للاكتشاف (0-100%)", + "settings.detectionGracePeriod": "فترة السماح للاكتشاف (ثوانٍ)", + "settings.detectionGracePeriodHelp": "الثواني بعد آخر اكتشاف قبل انتقال التسجيل إلى المخزن المؤقت اللاحق. زد هذه القيمة إذا كانت الاكتشافات تحتوي على فجوات قصيرة ينبغي التعامل معها كحركة مستمرة (0–60، الافتراضي: 2).", "settings.defaultPreDetectionBufferSeconds": "مخزن ما قبل الاكتشاف الافتراضي (ثوانٍ)", "settings.defaultPreDetectionBufferHelp": "عدد ثواني الفيديو المحفوظة قبل الاكتشاف", "settings.defaultPostDetectionBufferSeconds": "مخزن ما بعد الاكتشاف الافتراضي (ثوانٍ)", diff --git a/web/public/locales/de.json b/web/public/locales/de.json index 9bef57ef..2f3d8af8 100644 --- a/web/public/locales/de.json +++ b/web/public/locales/de.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "Von der Erkennung verwendetes Inferenz-Backend API", "settings.defaultDetectionThreshold": "Standarderkennungsschwelle", "settings.defaultDetectionThresholdHelp": "Vertrauensschwelle für die Erkennung (0–100 %)", + "settings.detectionGracePeriod": "Erkennungs-Toleranzzeit (Sekunden)", + "settings.detectionGracePeriodHelp": "Sekunden nach der letzten Erkennung, bevor die Aufnahme in den Nachpuffer wechselt. Erhöhen Sie diesen Wert, wenn Erkennungen kurze Lücken aufweisen, die als kontinuierliche Bewegung behandelt werden sollen (0–60, Standard: 2).", "settings.defaultPreDetectionBufferSeconds": "Standard-Vorerkennungspuffer (Sekunden)", "settings.defaultPreDetectionBufferHelp": "Sekundenlanges Video, das vor der Erkennung aufbewahrt werden muss", "settings.defaultPostDetectionBufferSeconds": "Standardpuffer nach der Erkennung (Sekunden)", diff --git a/web/public/locales/en.json b/web/public/locales/en.json index b60c96ca..dd761ee1 100644 --- a/web/public/locales/en.json +++ b/web/public/locales/en.json @@ -1065,6 +1065,8 @@ "settings.apiDetectionBackendHelp": "Inference backend used by the detection API", "settings.defaultDetectionThreshold": "Default Detection Threshold", "settings.defaultDetectionThresholdHelp": "Confidence threshold for detection (0-100%)", + "settings.detectionGracePeriod": "Detection Grace Period (seconds)", + "settings.detectionGracePeriodHelp": "Seconds after the last detection before recording transitions to post-buffer. Increase if detections have brief gaps that should be treated as continuous motion (0–60, default: 2).", "settings.defaultPreDetectionBufferSeconds": "Default Pre-detection Buffer (seconds)", "settings.defaultPreDetectionBufferHelp": "Seconds of video to keep before detection", "settings.defaultPostDetectionBufferSeconds": "Default Post-detection Buffer (seconds)", diff --git a/web/public/locales/es.json b/web/public/locales/es.json index d074a152..b91ac577 100644 --- a/web/public/locales/es.json +++ b/web/public/locales/es.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "Backend de inferencia utilizado por la detección API", "settings.defaultDetectionThreshold": "Umbral de detección predeterminado", "settings.defaultDetectionThresholdHelp": "Umbral de confianza para la detección (0-100%)", + "settings.detectionGracePeriod": "Período de gracia de detección (segundos)", + "settings.detectionGracePeriodHelp": "Segundos tras la última detección antes de que la grabación pase al búfer posterior. Auméntelo si las detecciones tienen breves interrupciones que deben tratarse como movimiento continuo (0–60, predeterminado: 2).", "settings.defaultPreDetectionBufferSeconds": "Búfer de detección previa predeterminado (segundos)", "settings.defaultPreDetectionBufferHelp": "Segundos de video para conservar antes de la detección", "settings.defaultPostDetectionBufferSeconds": "Búfer posterior a la detección predeterminado (segundos)", diff --git a/web/public/locales/fr.json b/web/public/locales/fr.json index 6dafe751..ce663179 100644 --- a/web/public/locales/fr.json +++ b/web/public/locales/fr.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "Backend d'inférence utilisé par la détection API", "settings.defaultDetectionThreshold": "Seuil de détection par défaut", "settings.defaultDetectionThresholdHelp": "Seuil de confiance pour la détection (0-100 %)", + "settings.detectionGracePeriod": "Délai de grâce de détection (secondes)", + "settings.detectionGracePeriodHelp": "Secondes après la dernière détection avant que l’enregistrement passe au tampon post-détection. Augmentez cette valeur si les détections ont de brèves interruptions devant être traitées comme un mouvement continu (0–60, par défaut : 2).", "settings.defaultPreDetectionBufferSeconds": "Tampon de pré-détection par défaut (secondes)", "settings.defaultPreDetectionBufferHelp": "Secondes de vidéo à conserver avant détection", "settings.defaultPostDetectionBufferSeconds": "Tampon de post-détection par défaut (secondes)", diff --git a/web/public/locales/it.json b/web/public/locales/it.json index 912a5c43..20103cdb 100644 --- a/web/public/locales/it.json +++ b/web/public/locales/it.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "Backend di inferenza utilizzato dal rilevamento API", "settings.defaultDetectionThreshold": "Soglia di rilevamento predefinita", "settings.defaultDetectionThresholdHelp": "Soglia di confidenza per il rilevamento (0-100%)", + "settings.detectionGracePeriod": "Periodo di tolleranza rilevamento (secondi)", + "settings.detectionGracePeriodHelp": "Secondi dopo l'ultimo rilevamento prima che la registrazione passi al buffer post-rilevamento. Aumentare se i rilevamenti presentano brevi lacune che devono essere trattate come movimento continuo (0–60, predefinito: 2).", "settings.defaultPreDetectionBufferSeconds": "Buffer di pre-rilevamento predefinito (secondi)", "settings.defaultPreDetectionBufferHelp": "Secondi di video da conservare prima del rilevamento", "settings.defaultPostDetectionBufferSeconds": "Buffer post-rilevamento predefinito (secondi)", diff --git a/web/public/locales/ja.json b/web/public/locales/ja.json index 88aa2b9f..320ff2f7 100644 --- a/web/public/locales/ja.json +++ b/web/public/locales/ja.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "検出で使用される推論バックエンド API", "settings.defaultDetectionThreshold": "デフォルトの検出しきい値", "settings.defaultDetectionThresholdHelp": "検出の信頼しきい値 (0 ～ 100%)", + "settings.detectionGracePeriod": "検出グレース期間（秒）", + "settings.detectionGracePeriodHelp": "最後の検出後、録画がポストバッファーに移行するまでの秒数。検出に短い間隔があり、連続した動きとして扱うべき場合は増やしてください（0～60、デフォルト: 2）。", "settings.defaultPreDetectionBufferSeconds": "デフォルトの事前検出バッファー (秒)", "settings.defaultPreDetectionBufferHelp": "検出されるまでに保存するビデオの秒数", "settings.defaultPostDetectionBufferSeconds": "デフォルトの検出後バッファ (秒)", diff --git a/web/public/locales/ko.json b/web/public/locales/ko.json index e4a113d8..39e038ad 100644 --- a/web/public/locales/ko.json +++ b/web/public/locales/ko.json @@ -1065,6 +1065,8 @@ "settings.apiDetectionBackendHelp": "감지 API에 사용되는 추론 백엔드", "settings.defaultDetectionThreshold": "기본 감지 임계값", "settings.defaultDetectionThresholdHelp": "감지 신뢰도 임계값 (0~100%)", + "settings.detectionGracePeriod": "감지 유예 기간 (초)", + "settings.detectionGracePeriodHelp": "마지막 감지 후 녹화가 사후 버퍼로 전환되기까지의 초 수. 감지에 연속 움직임으로 처리해야 할 짧은 간격이 있는 경우 늘리세요 (0–60, 기본값: 2).", "settings.defaultPreDetectionBufferSeconds": "기본 감지 전 버퍼 (초)", "settings.defaultPreDetectionBufferHelp": "감지 전 보존할 영상 시간 (초)", "settings.defaultPostDetectionBufferSeconds": "기본 감지 후 버퍼 (초)", diff --git a/web/public/locales/nl.json b/web/public/locales/nl.json index 089a6f40..c5c39264 100644 --- a/web/public/locales/nl.json +++ b/web/public/locales/nl.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "Inferentie-backend gebruikt door de detectie API", "settings.defaultDetectionThreshold": "Standaard detectiedrempel", "settings.defaultDetectionThresholdHelp": "Betrouwbaarheidsdrempel voor detectie (0-100%)", + "settings.detectionGracePeriod": "Detectie-uitlooptijd (seconden)", + "settings.detectionGracePeriodHelp": "Seconden na de laatste detectie voordat de opname overgaat naar de nabuffer. Verhoog dit als detecties korte onderbrekingen hebben die als continue beweging moeten worden behandeld (0–60, standaard: 2).", "settings.defaultPreDetectionBufferSeconds": "Standaard voordetectiebuffer (seconden)", "settings.defaultPreDetectionBufferHelp": "Aantal seconden video dat moet worden bewaard vóór detectie", "settings.defaultPostDetectionBufferSeconds": "Standaard post-detectiebuffer (seconden)", diff --git a/web/public/locales/pl.json b/web/public/locales/pl.json index e036df4a..ed8c43bd 100644 --- a/web/public/locales/pl.json +++ b/web/public/locales/pl.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "Backend wnioskowania używany przez detekcję API", "settings.defaultDetectionThreshold": "Domyślny próg wykrywania", "settings.defaultDetectionThresholdHelp": "Próg ufności dla wykrycia (0-100%)", + "settings.detectionGracePeriod": "Okres tolerancji detekcji (sekundy)", + "settings.detectionGracePeriodHelp": "Sekundy po ostatnim wykryciu przed przejściem nagrania do bufora końcowego. Zwiększ, jeśli detekcje mają krótkie przerwy, które powinny być traktowane jako ciągły ruch (0–60, domyślnie: 2).", "settings.defaultPreDetectionBufferSeconds": "Domyślny bufor wstępnego wykrywania (sekundy)", "settings.defaultPreDetectionBufferHelp": "Sekundy filmu, które należy zachować przed wykryciem", "settings.defaultPostDetectionBufferSeconds": "Domyślny bufor po wykryciu (sekundy)", diff --git a/web/public/locales/pt-BR.json b/web/public/locales/pt-BR.json index 593c66be..8eeb7601 100644 --- a/web/public/locales/pt-BR.json +++ b/web/public/locales/pt-BR.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "Backend de inferência usado pela API de detecção", "settings.defaultDetectionThreshold": "Limiar de detecção padrão", "settings.defaultDetectionThresholdHelp": "Limiar de confiança para detecção (0-100%)", + "settings.detectionGracePeriod": "Período de carência da detecção (segundos)", + "settings.detectionGracePeriodHelp": "Segundos após a última detecção antes de a gravação passar para o buffer pós-detecção. Aumente se as detecções tiverem breves lacunas que devem ser tratadas como movimento contínuo (0–60, padrão: 2).", "settings.defaultPreDetectionBufferSeconds": "Buffer padrão pré-detecção (segundos)", "settings.defaultPreDetectionBufferHelp": "Segundos de vídeo a manter antes da detecção", "settings.defaultPostDetectionBufferSeconds": "Buffer padrão pós-detecção (segundos)", diff --git a/web/public/locales/pt-PT.json b/web/public/locales/pt-PT.json index 65bc428d..0c29341e 100644 --- a/web/public/locales/pt-PT.json +++ b/web/public/locales/pt-PT.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "Backend de inferência usado pela API de deteção", "settings.defaultDetectionThreshold": "Limiar de deteção predefinido", "settings.defaultDetectionThresholdHelp": "Limiar de confiança para deteção (0-100%)", + "settings.detectionGracePeriod": "Período de tolerância da deteção (segundos)", + "settings.detectionGracePeriodHelp": "Segundos após a última deteção antes de a gravação transitar para o buffer pós-deteção. Aumente se as deteções tiverem breves lacunas que devem ser tratadas como movimento contínuo (0–60, predefinição: 2).", "settings.defaultPreDetectionBufferSeconds": "Buffer pré-deteção predefinido (segundos)", "settings.defaultPreDetectionBufferHelp": "Segundos de vídeo a manter antes da deteção", "settings.defaultPostDetectionBufferSeconds": "Buffer pós-deteção predefinido (segundos)", diff --git a/web/public/locales/ru.json b/web/public/locales/ru.json index c0484de4..118880e3 100644 --- a/web/public/locales/ru.json +++ b/web/public/locales/ru.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "Сервер вывода, используемый для обнаружения API", "settings.defaultDetectionThreshold": "Порог обнаружения по умолчанию", "settings.defaultDetectionThresholdHelp": "Порог уверенности для обнаружения (0–100 %)", + "settings.detectionGracePeriod": "Льготный период обнаружения (секунды)", + "settings.detectionGracePeriodHelp": "Секунды после последнего обнаружения до перехода записи в пост-буфер. Увеличьте значение, если обнаружения имеют короткие паузы, которые следует считать непрерывным движением (0–60, по умолчанию: 2).", "settings.defaultPreDetectionBufferSeconds": "Буфер предварительного обнаружения по умолчанию (секунды)", "settings.defaultPreDetectionBufferHelp": "Секунды видео, которые нужно сохранить до обнаружения", "settings.defaultPostDetectionBufferSeconds": "Буфер после обнаружения по умолчанию (секунды)", diff --git a/web/public/locales/tr.json b/web/public/locales/tr.json index 22ac20aa..04a8fdc1 100644 --- a/web/public/locales/tr.json +++ b/web/public/locales/tr.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "Algılama API tarafından kullanılan çıkarım arka ucu", "settings.defaultDetectionThreshold": "Varsayılan Algılama Eşiği", "settings.defaultDetectionThresholdHelp": "Tespit için güven eşiği (%0-100)", + "settings.detectionGracePeriod": "Algılama Tolerans Süresi (saniye)", + "settings.detectionGracePeriodHelp": "Son algılamadan sonra kaydın son arabelleğe geçmesine kadar geçen saniye sayısı. Algılamalar sürekli hareket olarak değerlendirilmesi gereken kısa aralıklara sahipse artırın (0–60, varsayılan: 2).", "settings.defaultPreDetectionBufferSeconds": "Varsayılan Ön Tespit Arabelleği (saniye)", "settings.defaultPreDetectionBufferHelp": "Tespit edilmeden önce saklanacak saniyelik video", "settings.defaultPostDetectionBufferSeconds": "Varsayılan Algılama Sonrası Ara Bellek (saniye)", diff --git a/web/public/locales/uk.json b/web/public/locales/uk.json index 0216a538..ee73ba4f 100644 --- a/web/public/locales/uk.json +++ b/web/public/locales/uk.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "Сервер виведення, який використовується виявленням API", "settings.defaultDetectionThreshold": "Поріг виявлення за замовчуванням", "settings.defaultDetectionThresholdHelp": "Достовірний поріг виявлення (0-100%)", + "settings.detectionGracePeriod": "Пільговий період виявлення (секунди)", + "settings.detectionGracePeriodHelp": "Секунди після останнього виявлення до переходу запису в пост-буфер. Збільшіть значення, якщо виявлення мають короткі паузи, які слід вважати безперервним рухом (0–60, за замовчуванням: 2).", "settings.defaultPreDetectionBufferSeconds": "Буфер попереднього виявлення за замовчуванням (секунди)", "settings.defaultPreDetectionBufferHelp": "Секунди відео, які потрібно зберегти до виявлення", "settings.defaultPostDetectionBufferSeconds": "Буфер після виявлення за замовчуванням (секунди)", diff --git a/web/public/locales/zh-TW.json b/web/public/locales/zh-TW.json index c4d486bd..16f3139e 100644 --- a/web/public/locales/zh-TW.json +++ b/web/public/locales/zh-TW.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "檢測所使用的推理後端API", "settings.defaultDetectionThreshold": "預設檢測閾值", "settings.defaultDetectionThresholdHelp": "檢測置信閾值（0-100%）", + "settings.detectionGracePeriod": "檢測寬限期（秒）", + "settings.detectionGracePeriodHelp": "最後一次檢測後錄製切換至後置緩衝區之前的秒數。若檢測存在短暫間隔但應視為連續動作，請增大此值（0–60，預設：2）。", "settings.defaultPreDetectionBufferSeconds": "預設預檢測緩衝區（秒）", "settings.defaultPreDetectionBufferHelp": "檢測前保留的影片秒數", "settings.defaultPostDetectionBufferSeconds": "預設檢測後緩衝區（秒）", diff --git a/web/public/locales/zh.json b/web/public/locales/zh.json index ba10aea6..13944919 100644 --- a/web/public/locales/zh.json +++ b/web/public/locales/zh.json @@ -1063,6 +1063,8 @@ "settings.apiDetectionBackendHelp": "检测使用的推理后端API", "settings.defaultDetectionThreshold": "默认检测阈值", "settings.defaultDetectionThresholdHelp": "检测置信阈值（0-100%）", + "settings.detectionGracePeriod": "检测宽限期（秒）", + "settings.detectionGracePeriodHelp": "最后一次检测后录制切换到后置缓冲区之前的秒数。如果检测存在短暂间隔但应视为连续运动，请增大此值（0–60，默认：2）。", "settings.defaultPreDetectionBufferSeconds": "默认预检测缓冲区（秒）", "settings.defaultPreDetectionBufferHelp": "检测前保留的视频秒数", "settings.defaultPostDetectionBufferSeconds": "默认检测后缓冲区（秒）", From adeca194ec4f537b365ae945edcb4577abe33402 Mon Sep 17 00:00:00 2001 From: Piotr Ziecik Date: Fri, 19 Jun 2026 10:47:20 +0200 Subject: [PATCH 07/13] feat(detection): record detections at frame-arrival time Capture the wall-clock time when a frame enters the detection pipeline and thread it through run_detection_on_frame() into detect_objects_api / detect_objects_api_snapshot, so detections.timestamp (and the MQTT event) reflect frame time rather than inference-completion time. ONVIF captures its timestamp before the PullMessages roundtrip. The DB layer still falls back to time(NULL) when callers pass 0, preserving the legacy detection.c stop-gap caller. Co-Authored-By: Claude Opus 4.8 (1M context) --- include/video/api_detection.h | 11 +++++-- include/video/unified_detection_thread.h | 1 + src/video/api_detection.c | 10 +++--- src/video/detection.c | 4 ++- src/video/onvif_detection.c | 15 ++++++--- src/video/unified_detection_thread.c | 42 ++++++++++++++++++------ tests/unit/test_api_detection.c | 2 +- 7 files changed, 62 insertions(+), 23 deletions(-) diff --git a/include/video/api_detection.h b/include/video/api_detection.h index 837788fa..959a56a7 100644 --- a/include/video/api_detection.h +++ b/include/video/api_detection.h @@ -3,6 +3,7 @@ #include #include +#include #include "video/detection_result.h" // Model type for API-based detection @@ -36,11 +37,16 @@ void shutdown_api_detection_system(void); * @param stream_name The name of the stream (for database storage) * @param threshold Confidence threshold for detection (0.0-1.0, use negative for default) * @param recording_id Recording ID to link detections to (0 for no link) + * @param frame_timestamp Wall-clock time when the frame entered the detection + * pipeline. Used as the DB/MQTT event timestamp so it + * reflects frame time rather than inference completion. + * Pass 0 to fall back to time(NULL) inside the DB layer. * @return 0 on success, non-zero on failure */ int detect_objects_api(const char *api_url, const unsigned char *frame_data, int width, int height, int channels, detection_result_t *result, - const char *stream_name, float threshold, uint64_t recording_id); + const char *stream_name, float threshold, uint64_t recording_id, + time_t frame_timestamp); /** * @brief Determine whether API detection should fetch a go2rtc snapshot. @@ -69,6 +75,7 @@ bool api_detection_should_use_go2rtc_snapshot(const unsigned char *frame_data, * @return 0 on success, -1 on general failure, -2 if go2rtc snapshot failed (caller should fall back) */ int detect_objects_api_snapshot(const char *api_url, const char *stream_name, - detection_result_t *result, float threshold, uint64_t recording_id); + detection_result_t *result, float threshold, + uint64_t recording_id, time_t frame_timestamp); #endif /* LIGHTNVR_API_DETECTION_H */ diff --git a/include/video/unified_detection_thread.h b/include/video/unified_detection_thread.h index 52a86390..0654ab74 100644 --- a/include/video/unified_detection_thread.h +++ b/include/video/unified_detection_thread.h @@ -173,6 +173,7 @@ typedef struct { atomic_int detection_stream_connected; // 1 = open & decoding, 0 = connecting/disconnected atomic_int detection_stream_result; // 1 = pending result ready, 0 = none detection_result_t detection_stream_pending; // latest result; guarded by result_mutex + time_t detection_stream_pending_ts; // frame time of the pending result (producer's detect time); guarded by result_mutex pthread_mutex_t detection_stream_result_mutex; // protects detection_stream_pending char detection_stream_url[MAX_URL_LENGTH]; // cached at thread-start diff --git a/src/video/api_detection.c b/src/video/api_detection.c index 19646ccb..790522bf 100644 --- a/src/video/api_detection.c +++ b/src/video/api_detection.c @@ -368,7 +368,8 @@ void shutdown_api_detection_system(void) { */ int detect_objects_api(const char *api_url, const unsigned char *frame_data, int width, int height, int channels, detection_result_t *result, - const char *stream_name, float threshold, uint64_t recording_id) { + const char *stream_name, float threshold, uint64_t recording_id, + time_t frame_timestamp) { // Check if we're in shutdown mode or if the stream has been stopped. if (is_shutdown_initiated()) { log_info("API Detection: System shutdown in progress, skipping detection"); @@ -715,7 +716,7 @@ int detect_objects_api(const char *api_url, const unsigned char *frame_data, filter_detections_by_stream_objects(stream_name, result); - time_t timestamp = time(NULL); + time_t timestamp = (frame_timestamp != 0) ? frame_timestamp : time(NULL); store_detections_in_db(stream_name, result, timestamp, recording_id); if (result->count > 0) { @@ -753,7 +754,8 @@ int detect_objects_api(const char *api_url, const unsigned char *frame_data, * Returns: 0 on success, -1 on general failure, -2 if go2rtc snapshot failed */ int detect_objects_api_snapshot(const char *api_url, const char *stream_name, - detection_result_t *result, float threshold, uint64_t recording_id) { + detection_result_t *result, float threshold, + uint64_t recording_id, time_t frame_timestamp) { // Check if we're in shutdown mode if (is_shutdown_initiated()) { log_info("API Detection (snapshot): System shutdown in progress, skipping detection"); @@ -1080,7 +1082,7 @@ int detect_objects_api_snapshot(const char *api_url, const char *stream_name, // Filter detections by per-stream object include/exclude lists filter_detections_by_stream_objects(stream_name, result); - time_t timestamp = time(NULL); + time_t timestamp = (frame_timestamp != 0) ? frame_timestamp : time(NULL); store_detections_in_db(stream_name, result, timestamp, recording_id); // Publish to MQTT if enabled diff --git a/src/video/detection.c b/src/video/detection.c index 32427b3b..d9f0946e 100644 --- a/src/video/detection.c +++ b/src/video/detection.c @@ -238,7 +238,9 @@ int detect_objects(detection_model_t model, const unsigned char *frame_data, // The stream name will be set by the caller when storing the detections // A negative threshold tells detect_objects_api to use the default (0.5) // recording_id of 0 means no recording linkage - ret = detect_objects_api(api_url, frame_data, width, height, channels, result, NULL, -1.0f, 0); + // Legacy stop-gap caller: no frame-arrival timestamp available + // here, so pass 0 to let the DB layer fall back to time(NULL). + ret = detect_objects_api(api_url, frame_data, width, height, channels, result, NULL, -1.0f, 0, 0); } } else { diff --git a/src/video/onvif_detection.c b/src/video/onvif_detection.c index ad88a9fa..a6d029d2 100644 --- a/src/video/onvif_detection.c +++ b/src/video/onvif_detection.c @@ -653,6 +653,12 @@ int detect_motion_onvif(const char *onvif_url, const char *username, const char " 100\n" ""; + // Capture the event timestamp BEFORE the (potentially up-to-5-second) + // PullMessages roundtrip and the subsequent zone-filter step. Storing + // this as the detection timestamp keeps detections.timestamp aligned with + // when the motion window started, not when inference + DB write finished. + time_t event_timestamp = time(NULL); + // Send PullMessages directly to the subscription address (the full URL returned by // CreatePullPointSubscription per ONVIF spec). Fall back to the legacy path-extraction // approach only when the stored address is not an absolute HTTP URL. @@ -718,13 +724,12 @@ int detect_motion_onvif(const char *onvif_url, const char *username, const char } // Store the detection in the database (no recording_id linkage for ONVIF) - time_t timestamp = time(NULL); - store_detections_in_db(stream_name, result, timestamp, 0); + store_detections_in_db(stream_name, result, event_timestamp, 0); // Publish to MQTT and trigger motion recording if detections remain after filtering if (result->count > 0) { - mqtt_publish_detection(stream_name, result, timestamp); - process_motion_event(stream_name, true, timestamp, false); + mqtt_publish_detection(stream_name, result, event_timestamp); + process_motion_event(stream_name, true, event_timestamp, false); } } else { log_warn("No stream name provided, skipping database storage"); @@ -735,7 +740,7 @@ int detect_motion_onvif(const char *onvif_url, const char *username, const char // Notify motion recording that motion has ended if (stream_name && stream_name[0] != '\0') { - process_motion_event(stream_name, false, time(NULL), false); + process_motion_event(stream_name, false, event_timestamp, false); } } diff --git a/src/video/unified_detection_thread.c b/src/video/unified_detection_thread.c index f68f726f..57a07f16 100644 --- a/src/video/unified_detection_thread.c +++ b/src/video/unified_detection_thread.c @@ -112,7 +112,8 @@ static void stop_detection_stream_thread(unified_detection_ctx_t *ctx); static bool detect_on_decoded_frame(unified_detection_ctx_t *ctx, AVFrame *frame, time_t now, detection_result_t *result); -static bool run_detection_on_frame(unified_detection_ctx_t *ctx, AVPacket *pkt); +static bool run_detection_on_frame(unified_detection_ctx_t *ctx, AVPacket *pkt, + time_t frame_timestamp); /* Action layer */ static void report_detections(unified_detection_ctx_t *ctx, const detection_result_t *result, time_t now); @@ -808,11 +809,16 @@ static void *detection_stream_thread_func(void *arg) { bool hit = detect_on_decoded_frame(ctx, frame, now, &result); if (hit) { - /* Write result to shared slot then raise the flag. - * process_packet() reads flag + slot and calls - * report_detections() + handle_recording_state(). */ + /* Write result + its frame time to the shared slot, then + * raise the flag. process_packet() reads flag + slot and + * calls report_detections() + handle_recording_state(). + * Carrying the timestamp keeps detections.timestamp at + * detection time rather than the consumer's later time — + * matching the API path, which writes the DB here with + * the same `now`. */ pthread_mutex_lock(&ctx->detection_stream_result_mutex); ctx->detection_stream_pending = result; + ctx->detection_stream_pending_ts = now; pthread_mutex_unlock(&ctx->detection_stream_result_mutex); atomic_store(&ctx->detection_stream_result, 1); } @@ -847,6 +853,7 @@ static int start_detection_stream_thread(unified_detection_ctx_t *ctx) { atomic_store(&ctx->detection_stream_connected, 0); atomic_store(&ctx->detection_stream_result, 0); memset(&ctx->detection_stream_pending, 0, sizeof(ctx->detection_stream_pending)); + ctx->detection_stream_pending_ts = 0; pthread_attr_t attr; pthread_attr_init(&attr); @@ -1054,6 +1061,7 @@ int start_unified_detection_thread(const char *stream_name, const char *model_pa atomic_store(&ctx->detection_stream_connected, 0); atomic_store(&ctx->detection_stream_result, 0); memset(&ctx->detection_stream_pending, 0, sizeof(ctx->detection_stream_pending)); + ctx->detection_stream_pending_ts = 0; // For ONVIF model: cache connection parameters and start the background // polling thread BEFORE the UDT starts reading packets. This ensures a @@ -2071,14 +2079,21 @@ static int process_packet(unified_detection_ctx_t *ctx, AVPacket *pkt) { bool detected = false; if (atomic_exchange(&ctx->detection_stream_result, 0)) { detection_result_t result; + time_t result_ts; pthread_mutex_lock(&ctx->detection_stream_result_mutex); result = ctx->detection_stream_pending; + result_ts = ctx->detection_stream_pending_ts; pthread_mutex_unlock(&ctx->detection_stream_result_mutex); log_info("[%s] Detection stream result: %d detection(s)", ctx->stream_name, result.count); - report_detections(ctx, &result, now); + /* Stamp the DB write with the producer's detection time (when the + * frame was actually analyzed), not this consumer's wall-clock — + * otherwise detections lag the video by the hand-off latency. */ + report_detections(ctx, &result, result_ts > 0 ? result_ts : now); detected = true; } + // handle_recording_state drives the recording state machine, which is a + // "now" decision (grace/post-buffer), so it keeps the consumer's now. handle_recording_state(ctx, detected, now); // Re-read state: handle_recording_state may have transitioned // BUFFERING → RECORDING or POST_BUFFER → RECORDING above. @@ -2219,7 +2234,7 @@ static int process_packet(unified_detection_ctx_t *ctx, AVPacket *pkt) { log_info("[%s] Running detection (interval=%ds, elapsed=%lds, model=%s)", ctx->stream_name, ctx->detection_interval, (long)time_since_last_check, ctx->model_path); - bool detected = run_detection_on_frame(ctx, pkt); + bool detected = run_detection_on_frame(ctx, pkt, now); handle_recording_state(ctx, detected, now); } } @@ -2495,10 +2510,15 @@ static int flush_prebuffer_to_recording(unified_detection_ctx_t *ctx) { * @param pkt The video packet containing a keyframe (unused for API detection) * @return true if detection was triggered, false otherwise */ -static bool run_detection_on_frame(unified_detection_ctx_t *ctx, AVPacket *pkt) { +static bool run_detection_on_frame(unified_detection_ctx_t *ctx, AVPacket *pkt, + time_t frame_timestamp) { if (!ctx) return false; - time_t now = time(NULL); + /* frame_timestamp reflects when the packet entered the pipeline + * (captured at the top of process_packet). Reuse it for reporting, + * DB writes, and MQTT so detections.timestamp records frame time + * rather than inference-finished time. */ + time_t now = frame_timestamp; /* API detection: try the go2rtc snapshot shortcut first. * The snapshot path handles zone filtering, DB storage, and MQTT @@ -2518,7 +2538,8 @@ static bool run_detection_on_frame(unified_detection_ctx_t *ctx, AVPacket *pkt) detection_result_t result; memset(&result, 0, sizeof(result)); int detect_ret = detect_objects_api_snapshot(ctx->model_path, ctx->stream_name, - &result, ctx->detection_threshold, rec_id); + &result, ctx->detection_threshold, + rec_id, frame_timestamp); if (detect_ret != DETECT_SNAPSHOT_UNAVAILABLE) { if (detect_ret != 0) { @@ -2649,7 +2670,8 @@ static bool detect_on_decoded_frame(unified_detection_ctx_t *ctx, const char *api_url = get_actual_api_url(ctx->stream_name, ctx->model_path); int ret = api_url ? detect_objects_api(api_url, rgb_buf, width, height, 3, - result, ctx->stream_name, ctx->detection_threshold, rec_id) + result, ctx->stream_name, ctx->detection_threshold, + rec_id, now) : -1; free(rgb_buf); diff --git a/tests/unit/test_api_detection.c b/tests/unit/test_api_detection.c index 7bf28959..011ff14e 100644 --- a/tests/unit/test_api_detection.c +++ b/tests/unit/test_api_detection.c @@ -23,7 +23,7 @@ static void assert_invalid_detection_url(const char *url) { detection_result_t result; memset(&result, 0xAB, sizeof(result)); - int rc = detect_objects_api(url, NULL, 0, 0, 0, &result, NULL, 0.5f, 0); + int rc = detect_objects_api(url, NULL, 0, 0, 0, &result, NULL, 0.5f, 0, 0); TEST_ASSERT_EQUAL_INT(-1, rc); TEST_ASSERT_EQUAL_INT(0, result.count); From 1ed572cfd651a0ca933ef7c17848b78ae6ae2af9 Mon Sep 17 00:00:00 2001 From: Piotr Ziecik Date: Fri, 19 Jun 2026 10:48:04 +0200 Subject: [PATCH 08/13] feat(detection): in-process LiteRT/TFLite detection engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the dead libtensorflowlite.so dlopen stub with a real in-process LiteRT engine. Streams with a .tflite model_path flow through the existing detection_model_t / detect_objects dispatch — no UDT sentinel. - src/video/detection/litert_engine.{h,cc}: opaque C ABI + refcounted per-model registry (engines shared across streams using the same .tflite, mutex-serialized). Data-driven input dims/dtype (float32/uint8/int8 with per-tensor quantization), letterbox via libswscale, end-to-end YOLO [1,N,6] output enforced at init. Labels from a .labels.txt sidecar with class_ fallback. - CMake: ENABLE_LITERT (default OFF), LITERT_WITH_XNNPACK/_GPU; conditional add_subdirectory of the vendored LiteRT submodule + tensorflow-lite link + HAVE_LITERT. litert_engine.cc pinned to C++17. - detection_model reshaped to hold the opaque engine; threshold is per-call. - New [detection_engine] INI section (enabled/threads/delegate) with a matching DetectionTab subsection, round-tripped via api_handlers_settings. - Hard stream error on model-load failure: one-shot handle_stream_error (STREAM_ERR_MODEL_LOAD) instead of swallow-and-retry; last_error_message surfaced as streams.error_message and rendered as a StreamCard tooltip. - All 17 locales gain settings.detectionEngine* and streams.errorModelLoad. - Embedded migration count includes 0042; engine migration regenerated. LiteRT vendored as a git submodule under third_party/litert. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitmodules | 3 + CMakeLists.txt | 72 ++ config/lightnvr.ini | 9 + include/core/config.h | 19 + include/video/detection/litert_engine.h | 76 +++ include/video/detection_model_internal.h | 13 +- include/video/stream_state.h | 20 + include/video/unified_detection_thread.h | 5 +- src/core/config.c | 49 ++ src/video/detection.c | 23 +- src/video/detection/litert_engine.cc | 624 ++++++++++++++++++ src/video/detection_model.c | 129 ++-- src/video/stream_state.c | 21 +- src/video/unified_detection_thread.c | 23 +- src/web/api_handlers_detection_models.c | 12 +- src/web/api_handlers_settings.c | 29 + src/web/api_handlers_streams_get.c | 20 + third_party/litert | 1 + web/js/components/preact/SettingsView.jsx | 10 + web/js/components/preact/StreamCard.jsx | 1 + .../preact/settings/DetectionTab.jsx | 57 ++ web/public/locales/ar.json | 9 + web/public/locales/de.json | 9 + web/public/locales/en.json | 9 + web/public/locales/es.json | 9 + web/public/locales/fr.json | 9 + web/public/locales/it.json | 9 + web/public/locales/ja.json | 9 + web/public/locales/ko.json | 9 + web/public/locales/nl.json | 9 + web/public/locales/pl.json | 9 + web/public/locales/pt-BR.json | 9 + web/public/locales/pt-PT.json | 9 + web/public/locales/ru.json | 9 + web/public/locales/tr.json | 9 + web/public/locales/uk.json | 9 + web/public/locales/zh-TW.json | 9 + web/public/locales/zh.json | 9 + 38 files changed, 1268 insertions(+), 101 deletions(-) create mode 100644 include/video/detection/litert_engine.h create mode 100644 src/video/detection/litert_engine.cc create mode 160000 third_party/litert diff --git a/.gitmodules b/.gitmodules index d338a47b..148373af 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = go2rtc url = https://github.com/opensensor/go2rtc.git branch = dev +[submodule "third_party/litert"] + path = third_party/litert + url = https://github.com/google-ai-edge/LiteRT.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 23abbac9..7aeb180d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,13 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) option(ENABLE_SOD "Enable SOD library for object detection" ON) option(SOD_DYNAMIC_LINK "Dynamically link SOD library instead of static linking" OFF) +# In-process LiteRT (TFLite) detection engine. Default OFF — when ON, the +# vendored LiteRT submodule at third_party/litert is built and linked, and +# the litert_engine module is compiled. +option(ENABLE_LITERT "Enable in-process LiteRT (TFLite) detection engine" ON) +option(LITERT_WITH_XNNPACK "Compile XNNPACK delegate into the LiteRT engine" ON) +option(LITERT_WITH_GPU "Compile GPU (OpenCL/GLES) delegate into the LiteRT engine" OFF) + # go2rtc integration options option(ENABLE_GO2RTC "Enable go2rtc integration for WebRTC streaming" ON) set(GO2RTC_BINARY_PATH "/usr/local/bin/go2rtc" CACHE STRING "Path to go2rtc binary") @@ -246,6 +253,57 @@ if(ENABLE_SOD) endif() endif() +# Set up LiteRT (in-process TFLite engine) if enabled +set(LITERT_ENGINE_SOURCES "") +if(ENABLE_LITERT) + set(LITERT_SUBDIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/litert/tflite") + if(NOT EXISTS "${LITERT_SUBDIR}/CMakeLists.txt") + message(FATAL_ERROR + "ENABLE_LITERT=ON but ${LITERT_SUBDIR} is missing. " + "Run: git submodule update --init --recursive third_party/litert") + endif() + + # Forward delegate selections to the LiteRT subproject. + set(TFLITE_ENABLE_XNNPACK ${LITERT_WITH_XNNPACK} CACHE BOOL "" FORCE) + set(TFLITE_ENABLE_GPU ${LITERT_WITH_GPU} CACHE BOOL "" FORCE) + # ruy is TFLite's ARM CPU GEMM backend; it runs any ops the XNNPACK delegate + # doesn't handle (falling back to it instead of the slower gemmlowp/Eigen path). + set(TFLITE_ENABLE_RUY ON CACHE BOOL "" FORCE) + set(TFLITE_BUILD_SHARED_LIB OFF CACHE BOOL "" FORCE) + + # Build XNNPACK without its hand-written assembly microkernels, so it uses the + # portable NEON-intrinsic kernels instead. XNNPACK's cortex_a53 asm f32 6x8 GEMM + # kernel corrupts an A-row pointer under a multi-threaded pthreadpool on aarch64 + # big.LITTLE, SIGSEGV-ing with threads>1 (root-caused from a core dump; present + # even at XNNPACK HEAD). Disabling the asm path makes multi-threaded inference + # stable; confirmed on target. Small single-thread perf cost vs tuned asm. + if(LITERT_WITH_XNNPACK) + set(XNNPACK_ENABLE_ASSEMBLY OFF CACHE BOOL "" FORCE) + endif() + + add_subdirectory(${LITERT_SUBDIR} EXCLUDE_FROM_ALL) + add_compile_definitions(HAVE_LITERT) + + if(LITERT_WITH_XNNPACK) + add_compile_definitions(HAVE_LITERT_XNNPACK) + endif() + if(LITERT_WITH_GPU) + add_compile_definitions(HAVE_LITERT_GPU) + endif() + + set(LITERT_ENGINE_SOURCES + "${CMAKE_CURRENT_SOURCE_DIR}/src/video/detection/litert_engine.cc" + ) + # TFLite public headers (and the Abseil/FlatBuffers they pull in) require + # C++17, but the project default is C++14. Linking the tensorflow-lite + # target propagates cxx_std_17, yet make the requirement explicit on our + # own engine source so it compiles correctly regardless of propagation. + set_source_files_properties(${LITERT_ENGINE_SOURCES} PROPERTIES CXX_STANDARD 17) + message(STATUS "LiteRT engine enabled (XNNPACK=${LITERT_WITH_XNNPACK}, GPU=${LITERT_WITH_GPU})") +else() + message(STATUS "LiteRT engine disabled (set -DENABLE_LITERT=ON to enable)") +endif() + # HTTP Backend Configuration - libuv + llhttp only message(STATUS "Using libuv + llhttp HTTP backend") add_definitions(-DHTTP_BACKEND_LIBUV) @@ -471,6 +529,7 @@ set(SOURCES ${HLS_UNIFIED_THREAD_SOURCES} ${TELEMETRY_SOURCES} ${EZXML_SOURCES} + ${LITERT_ENGINE_SOURCES} ) @@ -581,6 +640,14 @@ if(ENABLE_MQTT AND MOSQUITTO_FOUND) target_include_directories(lightnvr PRIVATE ${MOSQUITTO_INCLUDE_DIRS}) endif() +# Link LiteRT if enabled +if(ENABLE_LITERT) + # LiteRT's CMake target is named tensorflow-lite (provided by the LiteRT + # submodule's tflite/CMakeLists.txt). + target_link_libraries(lightnvr_lib PUBLIC tensorflow-lite) + target_link_libraries(lightnvr tensorflow-lite) +endif() + # Link SOD library if enabled if(ENABLE_SOD) # Always link to the sod target, whether it's built as static or shared @@ -683,6 +750,11 @@ if(ENABLE_SOD) message(STATUS " - SOD linking method: Static") endif() endif() +message(STATUS "- LiteRT (in-process TFLite engine): ${ENABLE_LITERT}") +if(ENABLE_LITERT) + message(STATUS " - XNNPACK delegate: ${LITERT_WITH_XNNPACK}") + message(STATUS " - GPU delegate: ${LITERT_WITH_GPU}") +endif() message(STATUS "- go2rtc WebRTC integration: ${ENABLE_GO2RTC}") if(ENABLE_GO2RTC) message(STATUS " - go2rtc binary path: ${GO2RTC_BINARY_PATH}") diff --git a/config/lightnvr.ini b/config/lightnvr.ini index cd6ce8c0..26ea85ca 100755 --- a/config/lightnvr.ini +++ b/config/lightnvr.ini @@ -49,6 +49,15 @@ backend = onnx ; Detection backend: onnx (YOLOv8 - best accuracy), tflite, or o confidence_threshold = 0.35 ; Lower threshold to catch distant vehicles filter_classes = car,motorcycle,truck,bus,bicycle ; Vehicle classes only +[detection_engine] +; In-process TFLite/LiteRT inference. When enabled, any stream whose +; model_path ends in .tflite will route through this engine. Requires +; LightNVR built with -DENABLE_LITERT=ON. Labels are read from the model's +; embedded TFLite metadata or a sidecar .labels.txt. +enabled = false +threads = 1 ; Interpreter threads (1-16); raise to use multiple CPU cores +delegate = xnnpack ; xnnpack | gpu | none (gpu requires -DLITERT_WITH_GPU=ON; falls back to CPU otherwise) + [memory] buffer_size = 1024 ; Buffer size in KB use_swap = true diff --git a/include/core/config.h b/include/core/config.h index 2537d24a..cb521859 100644 --- a/include/core/config.h +++ b/include/core/config.h @@ -129,6 +129,14 @@ typedef struct { // Size of recording schedule text buffer: 168 values + 167 commas + null terminator #define RECORDING_SCHEDULE_TEXT_SIZE 512 +// In-process LiteRT (TFLite) detection engine — runtime knobs only. +// Per-model state (model_path, labels) is configured per-stream, not here. +typedef struct { + bool enabled; // Master toggle (default: false) + int num_threads; // Interpreter::SetNumThreads, clamped 1..16 (default: 1) + char delegate[16]; // "xnnpack" | "gpu" | "none" (default: "xnnpack") +} detection_engine_config_t; + // Main configuration structure typedef struct { // General settings @@ -271,6 +279,10 @@ typedef struct { bool mqtt_ha_discovery; // Enable HA MQTT auto-discovery (default: false) char mqtt_ha_discovery_prefix[128]; // HA discovery topic prefix (default: "homeassistant") int mqtt_ha_snapshot_interval; // Snapshot publish interval in seconds (default: 30, 0=disabled) + + // In-process LiteRT (TFLite) detection engine. + // Runtime knobs only; per-model state (path, labels) lives elsewhere. + detection_engine_config_t detection_engine; } config_t; /** @@ -317,6 +329,13 @@ void load_default_config(config_t *config); */ void config_set_detection_grace_period(config_t *config, int seconds); +/** Set the LiteRT interpreter thread count, clamped to 1..16. */ +void config_set_detection_engine_threads(config_t *config, int threads); + +/** Set the LiteRT delegate if it is one of "xnnpack"/"gpu"/"none"; returns + * true if accepted, false (leaving the config unchanged) otherwise. */ +bool config_set_detection_engine_delegate(config_t *config, const char *delegate); + /** * Validate and normalize configuration values * diff --git a/include/video/detection/litert_engine.h b/include/video/detection/litert_engine.h new file mode 100644 index 00000000..b3ae08e3 --- /dev/null +++ b/include/video/detection/litert_engine.h @@ -0,0 +1,76 @@ +/** + * litert_engine.h + * + * C ABI for the in-process LiteRT (TFLite) detection engine. + * + * Consumed only by src/video/detection_model.c and src/video/detection.c. + * Implementation in litert_engine.cc. + * + * All entry points are no-ops returning NULL/-1 when ENABLE_LITERT=OFF + * (HAVE_LITERT undefined). Callers must still gate calls with + * #ifdef HAVE_LITERT to avoid unresolved-symbol linkage. + */ +#ifndef LITERT_ENGINE_H +#define LITERT_ENGINE_H + +#include +#include + +#include "video/detection_result.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct litert_engine litert_engine_t; + +/* Acquire (or create) a refcounted engine for the given model file. + * Multiple acquires of the same canonical path return the same engine, + * with refcount bumped. Returns NULL on failure (errors logged). + * + * Failure modes (each logs a clear message before returning NULL): + * - g_config.detection_engine.enabled == false + * - model file missing or unreadable + * - flatbuffer parse failure + * - unsupported input/output dtype (only float32/uint8/int8) + * - unsupported output shape (must be [1, N, 6]) + * - delegate ModifyGraphWithDelegate failure (after CPU fallback attempt) + */ +litert_engine_t *litert_engine_acquire(const char *model_path); + +/* Decrement refcount; when it reaches 0, destroy the engine and release + * delegate resources. Safe to call with NULL. */ +void litert_engine_release(litert_engine_t *engine); + +/* Model input dimensions, as read from the .tflite at init time. + * Returns 0 if engine is NULL. */ +int litert_engine_input_width(const litert_engine_t *engine); +int litert_engine_input_height(const litert_engine_t *engine); + +/* Run inference on a single RGB24 frame (packed HxWx3 in original frame + * coordinates). The engine letterboxes internally to its model HxW. + * + * Detection count is written to out->count. Boxes are in + * out->detections[*].x/y/width/height as normalized 0..1 in the original + * frame coordinate space. Labels are read from a .labels.txt + * sidecar next to the model, falling back to "class_". + * + * per_stream_threshold is the only confidence cutoff (no engine-side + * floor). Range 0.0..1.0; the engine internally clamps to [0, 1]. + * + * Returns 0 on success (matching detect_objects() / detect_with_sod_model() + * convention — non-zero is treated as failure by the UDT), or -1 on + * error. On success out->count is 0 when no objects pass the threshold. + * + * Thread-safe — each engine has its own mutex; concurrent calls + * serialize through it. */ +int litert_engine_detect(litert_engine_t *engine, + const uint8_t *rgb, int w, int h, + float per_stream_threshold, + detection_result_t *out); + +#ifdef __cplusplus +} +#endif + +#endif /* LITERT_ENGINE_H */ diff --git a/include/video/detection_model_internal.h b/include/video/detection_model_internal.h index d2088363..d11b959b 100644 --- a/include/video/detection_model_internal.h +++ b/include/video/detection_model_internal.h @@ -16,14 +16,15 @@ /* ------------------------------------------------------------------ * TFLite sub-structure (embedded in model_t union) + * + * Refcounted engine pointer (litert_engine_t*) owned by the LiteRT + * registry in src/video/detection/litert_engine.cc. The per-stream + * confidence threshold lives in the shared model_t::threshold field + * (set at load time, passed to litert_engine_detect() by detection.c), + * so this sub-struct only needs the engine handle. * ------------------------------------------------------------------ */ typedef struct { - void *handle; /* Dynamic library handle */ - void *model; /* TFLite model handle */ - float threshold; /* Detection threshold */ - void *(*load_model)(const char *); - void (*free_model)(void *); - void *(*detect)(void *, const unsigned char *, int, int, int, int *, float); + void *engine; /* litert_engine_t* (opaque) */ } tflite_model_t; /* ------------------------------------------------------------------ diff --git a/include/video/stream_state.h b/include/video/stream_state.h index 5620c82e..8c141b51 100644 --- a/include/video/stream_state.h +++ b/include/video/stream_state.h @@ -20,6 +20,22 @@ typedef enum { STREAM_STATE_RECONNECTING // Stream is attempting to reconnect } stream_state_t; +/** + * Stream error code categories — passed to handle_stream_error(). + * + * Negative POSIX errno values (-ETIMEDOUT, -EINVAL, -EIO) are also accepted + * and classified into "timeout"/"protocol"/"io" buckets in the telemetry + * metrics. The positive STREAM_ERR_* values below are reserved for + * application-level conditions that should NOT trigger reconnect (the + * problem is permanent until config or operator action fixes it). + */ +#define STREAM_ERR_MODEL_LOAD 1 /* Detection model failed to load (missing file, bad shape, etc.) */ + +/* Length of the optional human-readable error message attached to a stream + * state manager when in STREAM_STATE_ERROR. Sized to fit a path plus a + * short prefix without truncation in most cases. */ +#define STREAM_ERROR_MESSAGE_MAX 256 + /** * Stream feature flags - represents which features are enabled for a stream */ @@ -98,6 +114,10 @@ typedef struct { // Callback management bool callbacks_enabled; // Whether callbacks are enabled + + // Human-readable error message set when handle_stream_error() drives the + // stream into STREAM_STATE_ERROR. Empty string in other states. + char last_error_message[STREAM_ERROR_MESSAGE_MAX]; } stream_state_manager_t; /** diff --git a/include/video/unified_detection_thread.h b/include/video/unified_detection_thread.h index 0654ab74..5a91d086 100644 --- a/include/video/unified_detection_thread.h +++ b/include/video/unified_detection_thread.h @@ -92,7 +92,10 @@ typedef struct { atomic_llong post_buffer_end_time; // When post-buffer recording should end atomic_int log_counter; // Counter for periodic logging; intentionally accessed without ctx->mutex, // but all accesses must use atomic operations, and exact accuracy is not critical. - + bool model_load_failed; // Set on first load_detection_model() failure to suppress + // per-frame retry/log spam. Once true, the stream has been + // driven into STREAM_STATE_ERROR via handle_stream_error(). + // Connection state atomic_int_fast64_t last_packet_time; atomic_int consecutive_failures; diff --git a/src/core/config.c b/src/core/config.c index 1434ef6a..4a549d89 100644 --- a/src/core/config.c +++ b/src/core/config.c @@ -348,6 +348,11 @@ void load_default_config(config_t *config) { config->detection_grace_period = 2; // 2 seconds grace before post-buffer safe_strcpy(config->default_buffer_strategy, "auto", 32, 0); // Auto-select buffer strategy + // In-process LiteRT detection engine defaults + config->detection_engine.enabled = false; + config->detection_engine.num_threads = 1; + safe_strcpy(config->detection_engine.delegate, "xnnpack", 16, 0); + // Database settings safe_strcpy(config->db_path, "/var/lib/lightnvr/lightnvr.db", MAX_PATH_LENGTH, 0); config->db_backup_interval_minutes = 60; @@ -604,6 +609,23 @@ void config_set_detection_grace_period(config_t *config, int seconds) { config->detection_grace_period = seconds; } +void config_set_detection_engine_threads(config_t *config, int threads) { + if (threads < 1) threads = 1; + if (threads > 16) threads = 16; + config->detection_engine.num_threads = threads; +} + +bool config_set_detection_engine_delegate(config_t *config, const char *delegate) { + if (!delegate || + (strcmp(delegate, "xnnpack") != 0 && strcmp(delegate, "gpu") != 0 && + strcmp(delegate, "none") != 0)) { + return false; + } + safe_strcpy(config->detection_engine.delegate, delegate, + sizeof(config->detection_engine.delegate), 0); + return true; +} + // Handler function for inih static int config_ini_handler(void* user, const char* section, const char* name, const char* value) { config_t* config = (config_t*)user; @@ -697,6 +719,22 @@ static int config_ini_handler(void* user, const char* section, const char* name, safe_strcpy(config->default_buffer_strategy, value, sizeof(config->default_buffer_strategy), 0); } } + // In-process LiteRT (TFLite) detection engine settings + else if (strcmp(section, "detection_engine") == 0) { + if (strcmp(name, "enabled") == 0) { + config->detection_engine.enabled = (strcmp(value, "true") == 0 || + strcmp(value, "1") == 0 || + strcmp(value, "yes") == 0 || + strcmp(value, "on") == 0); + } else if (strcmp(name, "threads") == 0) { + config_set_detection_engine_threads(config, safe_atoi(value, 1)); + } else if (strcmp(name, "delegate") == 0) { + if (!config_set_detection_engine_delegate(config, value)) { + log_warn("Unknown detection_engine.delegate '%s'; using xnnpack", value); + config_set_detection_engine_delegate(config, "xnnpack"); + } + } + } // General detection behaviour settings (apply to all detection backends) else if (strcmp(section, "detection") == 0) { if (strcmp(name, "grace_period") == 0) { @@ -1623,6 +1661,17 @@ int save_config(const config_t *config, const char *path) { fprintf(file, "post_detection_buffer = %d\n", config->default_post_detection_buffer); fprintf(file, "buffer_strategy = %s\n\n", config->default_buffer_strategy); + // Write in-process LiteRT detection engine settings + fprintf(file, "[detection_engine]\n"); + fprintf(file, "; In-process TFLite/LiteRT inference. Streams using a .tflite model_path\n"); + fprintf(file, "; will route through this engine. Per-model labels are read from embedded\n"); + fprintf(file, "; TFLite metadata or a sidecar .labels.txt next to the model.\n"); + fprintf(file, "enabled = %s\n", config->detection_engine.enabled ? "true" : "false"); + fprintf(file, "threads = %d ; Interpreter threads (1-16, default: 4)\n", + config->detection_engine.num_threads); + fprintf(file, "delegate = %s ; xnnpack | gpu | none (default: xnnpack; gpu requires -DLITERT_WITH_GPU=ON)\n\n", + config->detection_engine.delegate); + // Write database settings fprintf(file, "[database]\n"); fprintf(file, "path = %s\n", config->db_path); diff --git a/src/video/detection.c b/src/video/detection.c index d9f0946e..1219a786 100644 --- a/src/video/detection.c +++ b/src/video/detection.c @@ -20,7 +20,11 @@ #include "../../include/video/detection.h" #include "../../include/video/detection_model.h" +#include "../../include/video/detection_model_internal.h" #include "../../include/video/sod_detection.h" +#ifdef HAVE_LITERT +#include "video/detection/litert_engine.h" +#endif #include "../../include/video/sod_realnet.h" #include "../../include/video/motion_detection.h" #include "../../include/video/api_detection.h" @@ -224,8 +228,25 @@ int detect_objects(detection_model_t model, const unsigned char *frame_data, } } else if (strcmp(model_type, MODEL_TYPE_TFLITE) == 0) { - log_error("TFLite detection not implemented yet"); +#ifdef HAVE_LITERT + model_t *m = (model_t *)model; + if (channels != 3) { + log_error("LiteRT engine requires RGB24 input (channels=3), got %d", channels); + ret = -1; + } else if (!m->tflite.engine) { + log_error("LiteRT engine handle is NULL for model %s", m->path); + ret = -1; + } else { + ret = litert_engine_detect(m->tflite.engine, frame_data, width, height, + m->threshold, result); + if (ret != 0) { + log_error("LiteRT inference failed for %s", m->path); + } + } +#else + log_error("LiteRT not compiled in (rebuild with -DENABLE_LITERT=ON)"); ret = -1; +#endif } else if (strcmp(model_type, MODEL_TYPE_API) == 0) { // For API models, the model_path contains the API URL diff --git a/src/video/detection/litert_engine.cc b/src/video/detection/litert_engine.cc new file mode 100644 index 00000000..851b4837 --- /dev/null +++ b/src/video/detection/litert_engine.cc @@ -0,0 +1,624 @@ +/** + * litert_engine.cc + * + * In-process LiteRT (TFLite) detection engine. + * + * - Refcounted registry keyed by canonical model path (one engine per + * unique .tflite, shared across streams that use the same model). + * - Per-engine std::mutex for inference serialization (tflite::Interpreter + * is not thread-safe). + * - Tensor introspection at init: input dims/dtype, output dims/dtype, and + * per-tensor quantization params (scale, zero_point) are read from the + * model — nothing hardcoded. + * - Supports float32, uint8, and int8 input/output dtypes. Any other type + * (e.g. float16, int16) causes init to fail with a clear error. + * - End-to-end YOLO output: [1, N, 6] = (x1, y1, x2, y2, conf, class_id). + * Output rank/shape mismatch causes init to fail. + * - Letterbox preprocess preserving aspect ratio, pad with 114 (Ultralytics + * convention). Uses libswscale for the resize. + * - Labels (v1): sidecar `.labels.txt` next to the .tflite, with + * synthetic `class_` fallback. Embedded TFLite metadata reading + * (Ultralytics convention) is a documented follow-up. + */ + +extern "C" { +#include "video/detection/litert_engine.h" +#include "core/config.h" +#include "core/logger.h" + +#include +#include +} + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_LITERT + +#include "tensorflow/lite/interpreter.h" +#include "tensorflow/lite/kernels/register.h" +#include "tensorflow/lite/model_builder.h" + +#ifdef HAVE_LITERT_XNNPACK +#include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h" +#endif + +#ifdef HAVE_LITERT_GPU +#include "tensorflow/lite/delegates/gpu/delegate.h" +#endif + +extern config_t g_config; + +namespace { + +constexpr int kExpectedOutputRank = 3; // [1, N, 6] +constexpr int kExpectedOutputCols = 6; // (x1, y1, x2, y2, conf, cls) +constexpr int kExpectedInputRank = 4; // [1, H, W, 3] +constexpr int kExpectedInputCh = 3; +constexpr uint8_t kLetterboxFill = 114; // Ultralytics convention + +struct engine_impl { + std::string canonical_path; + std::unique_ptr model; + std::unique_ptr interpreter; + TfLiteDelegate *delegate = nullptr; // owned; freed in dtor + void (*delegate_deleter)(TfLiteDelegate *) = nullptr; + + std::mutex inference_mu; + int refcount = 0; + + int input_w = 0; + int input_h = 0; + TfLiteType input_type = kTfLiteNoType; + TfLiteType output_type = kTfLiteNoType; + float input_scale = 0.0f; + int32_t input_zp = 0; + float output_scale = 0.0f; + int32_t output_zp = 0; + int output_num_rows = 0; // N in [1, N, 6] + + std::vector labels; + + ~engine_impl() { + // Interpreter must be destroyed before the delegate it references. + interpreter.reset(); + if (delegate && delegate_deleter) { + delegate_deleter(delegate); + } + } +}; + +std::mutex g_registry_mu; +std::unordered_map> g_registry; + +std::string canonicalize_path(const char *p) { + if (!p || !*p) return {}; + char buf[PATH_MAX]; + if (realpath(p, buf) != nullptr) return std::string(buf); + return std::string(p); // best-effort; engine will fail at file open +} + +std::vector load_labels_sidecar(const std::string &model_path) { + std::vector out; + + // Strip the .tflite extension and try .labels.txt + std::string base = model_path; + auto dot = base.find_last_of('.'); + if (dot != std::string::npos) base.resize(dot); + std::string sidecar = base + ".labels.txt"; + + std::ifstream f(sidecar); + if (!f.is_open()) { + log_info("LiteRT: no sidecar labels file at %s", sidecar.c_str()); + return out; + } + std::string line; + while (std::getline(f, line)) { + // strip trailing CR (Windows line endings) + while (!line.empty() && (line.back() == '\r' || line.back() == '\n')) { + line.pop_back(); + } + if (!line.empty()) out.push_back(line); + } + log_info("LiteRT: loaded %zu labels from %s", out.size(), sidecar.c_str()); + return out; +} + +// TODO(litert): read labels from embedded TFLite Metadata (Ultralytics +// `associated_files` convention). Until then, users of Ultralytics-exported +// models without a sidecar file get "class_" labels. + +const char *tflite_type_name(TfLiteType t) { + switch (t) { + case kTfLiteFloat32: return "float32"; + case kTfLiteUInt8: return "uint8"; + case kTfLiteInt8: return "int8"; + case kTfLiteFloat16: return "float16"; + case kTfLiteInt16: return "int16"; + case kTfLiteInt32: return "int32"; + default: return "?"; + } +} + +bool supported_io_dtype(TfLiteType t) { + return t == kTfLiteFloat32 || t == kTfLiteUInt8 || t == kTfLiteInt8; +} + +struct DelegateChoice { + TfLiteDelegate *delegate; + void (*deleter)(TfLiteDelegate *); + const char *name; +}; + +DelegateChoice make_delegate(const char *requested) { + DelegateChoice empty = {nullptr, nullptr, "none"}; + if (!requested || !*requested) return empty; + + if (strcmp(requested, "xnnpack") == 0) { +#ifdef HAVE_LITERT_XNNPACK + TfLiteXNNPackDelegateOptions opts = TfLiteXNNPackDelegateOptionsDefault(); + opts.num_threads = std::max(1, g_config.detection_engine.num_threads); + TfLiteDelegate *d = TfLiteXNNPackDelegateCreate(&opts); + if (d) return {d, &TfLiteXNNPackDelegateDelete, "xnnpack"}; + log_warn("LiteRT: XNNPACK delegate create failed; falling back to CPU"); + return empty; +#else + log_warn("LiteRT: delegate=xnnpack requested but not compiled in; using CPU"); + return empty; +#endif + } + if (strcmp(requested, "gpu") == 0) { +#ifdef HAVE_LITERT_GPU + TfLiteGpuDelegateOptionsV2 opts = TfLiteGpuDelegateOptionsV2Default(); + opts.inference_priority1 = TFLITE_GPU_INFERENCE_PRIORITY_MIN_LATENCY; + TfLiteDelegate *d = TfLiteGpuDelegateV2Create(&opts); + if (d) return {d, &TfLiteGpuDelegateV2Delete, "gpu"}; + log_warn("LiteRT: GPU delegate create failed; falling back to CPU"); + return empty; +#else + log_warn("LiteRT: delegate=gpu requested but not compiled in; using CPU"); + return empty; +#endif + } + if (strcmp(requested, "none") == 0) return empty; + log_warn("LiteRT: unknown delegate '%s'; using CPU", requested); + return empty; +} + +// Initialize an engine_impl from a model path. On failure, logs the cause +// and returns nullptr. On success the engine is fully ready for detect(). +std::unique_ptr create_engine(const std::string &canon_path) { + auto e = std::make_unique(); + e->canonical_path = canon_path; + + e->model = tflite::FlatBufferModel::BuildFromFile(canon_path.c_str()); + if (!e->model) { + log_error("LiteRT: failed to load model file %s", canon_path.c_str()); + return nullptr; + } + + tflite::ops::builtin::BuiltinOpResolver resolver; + tflite::InterpreterBuilder builder(*e->model, resolver); + builder(&e->interpreter); + if (!e->interpreter) { + log_error("LiteRT: InterpreterBuilder failed for %s", canon_path.c_str()); + return nullptr; + } + + int threads = std::max(1, std::min(16, g_config.detection_engine.num_threads)); + e->interpreter->SetNumThreads(threads); + + // Delegate selection. ModifyGraphWithDelegate must happen BEFORE + // AllocateTensors, otherwise the delegate sees no graph. + DelegateChoice dc = make_delegate(g_config.detection_engine.delegate); + if (dc.delegate) { + if (e->interpreter->ModifyGraphWithDelegate(dc.delegate) != kTfLiteOk) { + log_warn("LiteRT: ModifyGraphWithDelegate failed for delegate=%s; falling back to CPU", + dc.name); + if (dc.deleter) dc.deleter(dc.delegate); + } else { + e->delegate = dc.delegate; + e->delegate_deleter = dc.deleter; + } + } + + if (e->interpreter->AllocateTensors() != kTfLiteOk) { + log_error("LiteRT: AllocateTensors failed for %s", canon_path.c_str()); + return nullptr; + } + + // Inspect input tensor. + if (e->interpreter->inputs().size() != 1) { + log_error("LiteRT: model has %zu inputs (expected 1)", + e->interpreter->inputs().size()); + return nullptr; + } + const TfLiteTensor *in = e->interpreter->input_tensor(0); + if (!in || !in->dims || in->dims->size != kExpectedInputRank) { + log_error("LiteRT: input tensor has unexpected rank (got %d, expected %d)", + in && in->dims ? in->dims->size : -1, kExpectedInputRank); + return nullptr; + } + if (in->dims->data[0] != 1 || in->dims->data[3] != kExpectedInputCh) { + log_error("LiteRT: input shape [%d,%d,%d,%d] not [1,H,W,3]", + in->dims->data[0], in->dims->data[1], in->dims->data[2], + in->dims->data[3]); + return nullptr; + } + e->input_h = in->dims->data[1]; + e->input_w = in->dims->data[2]; + e->input_type = in->type; + if (!supported_io_dtype(e->input_type)) { + log_error("LiteRT: unsupported input dtype %s (supported: float32, uint8, int8)", + tflite_type_name(e->input_type)); + return nullptr; + } + e->input_scale = in->params.scale; + e->input_zp = in->params.zero_point; + + // Inspect output tensor. + if (e->interpreter->outputs().size() != 1) { + log_error("LiteRT: model has %zu outputs (expected 1 end-to-end)", + e->interpreter->outputs().size()); + return nullptr; + } + const TfLiteTensor *out = e->interpreter->output_tensor(0); + if (!out || !out->dims || out->dims->size != kExpectedOutputRank) { + log_error("LiteRT: output tensor has unexpected rank (got %d, expected %d for end-to-end YOLO)", + out && out->dims ? out->dims->size : -1, kExpectedOutputRank); + return nullptr; + } + if (out->dims->data[0] != 1 || out->dims->data[2] != kExpectedOutputCols) { + log_error("LiteRT: output shape [%d,%d,%d] not [1,N,6] (end-to-end YOLO required)", + out->dims->data[0], out->dims->data[1], out->dims->data[2]); + return nullptr; + } + e->output_num_rows = out->dims->data[1]; + e->output_type = out->type; + if (!supported_io_dtype(e->output_type)) { + log_error("LiteRT: unsupported output dtype %s", tflite_type_name(e->output_type)); + return nullptr; + } + e->output_scale = out->params.scale; + e->output_zp = out->params.zero_point; + + // Labels. + e->labels = load_labels_sidecar(canon_path); + + log_info("LiteRT engine ready: model=%s input=%dx%d/%s output=[1,%d,6]/%s " + "threads=%d delegate=%s classes=%zu", + canon_path.c_str(), e->input_w, e->input_h, tflite_type_name(e->input_type), + e->output_num_rows, tflite_type_name(e->output_type), + threads, e->delegate ? "active" : "cpu", e->labels.size()); + + return e; +} + +// Letterbox an RGB24 frame into the engine's input tensor, recording the +// transform for later inverse mapping. +struct LetterboxParams { + float scale; + int pad_x; + int pad_y; + int new_w; + int new_h; +}; + +// Resize+pad src (w,h) into dst (model_w, model_h) RGB24, fill background +// with 114. Returns the transform parameters. +bool letterbox_rgb24(const uint8_t *src, int w, int h, + uint8_t *dst, int model_w, int model_h, + LetterboxParams *params) +{ + float sx = static_cast(model_w) / w; + float sy = static_cast(model_h) / h; + float scale = std::min(sx, sy); + int new_w = std::max(1, static_cast(std::lround(w * scale))); + int new_h = std::max(1, static_cast(std::lround(h * scale))); + int pad_x = (model_w - new_w) / 2; + int pad_y = (model_h - new_h) / 2; + + // Background fill. + memset(dst, kLetterboxFill, static_cast(model_w) * model_h * 3); + + // Build a one-shot sws context. For latency-sensitive use we could + // cache per-engine; v1 keeps it simple. + SwsContext *sws = sws_getContext(w, h, AV_PIX_FMT_RGB24, + new_w, new_h, AV_PIX_FMT_RGB24, + SWS_BILINEAR, nullptr, nullptr, nullptr); + if (!sws) { + log_error("LiteRT: sws_getContext failed (%dx%d -> %dx%d)", w, h, new_w, new_h); + return false; + } + + // Scratch buffer for the resized image, then we blit into the + // padded dst at (pad_x, pad_y). + std::vector resized(static_cast(new_w) * new_h * 3); + const uint8_t *src_data[1] = { src }; + int src_lines[1] = { w * 3 }; + uint8_t *dst_data[1] = { resized.data() }; + int dst_lines[1] = { new_w * 3 }; + sws_scale(sws, src_data, src_lines, 0, h, dst_data, dst_lines); + sws_freeContext(sws); + + for (int row = 0; row < new_h; ++row) { + uint8_t *dst_row = dst + ((pad_y + row) * model_w + pad_x) * 3; + const uint8_t *src_row = resized.data() + row * new_w * 3; + memcpy(dst_row, src_row, static_cast(new_w) * 3); + } + + params->scale = scale; + params->pad_x = pad_x; + params->pad_y = pad_y; + params->new_w = new_w; + params->new_h = new_h; + return true; +} + +// Fill the model input tensor with the letterboxed RGB24 buffer, dtype-aware. +void fill_input_tensor(engine_impl *e, const uint8_t *lb_rgb) +{ + const int n = e->input_w * e->input_h * 3; + switch (e->input_type) { + case kTfLiteFloat32: { + float *p = e->interpreter->typed_input_tensor(0); + for (int i = 0; i < n; ++i) p[i] = lb_rgb[i] / 255.0f; + break; + } + case kTfLiteUInt8: { + uint8_t *p = e->interpreter->typed_input_tensor(0); + if (e->input_scale > 0.0f && std::fabs(e->input_scale - 1.0f/255.0f) > 1e-6) { + // Non-identity uint8 quantization. q = pixel/255/scale + zp + for (int i = 0; i < n; ++i) { + float v = lb_rgb[i] / 255.0f / e->input_scale + e->input_zp; + int q = static_cast(std::lround(v)); + p[i] = static_cast(std::clamp(q, 0, 255)); + } + } else { + memcpy(p, lb_rgb, n); + } + break; + } + case kTfLiteInt8: { + int8_t *p = e->interpreter->typed_input_tensor(0); + float scale = (e->input_scale > 0.0f) ? e->input_scale : (1.0f / 255.0f); + for (int i = 0; i < n; ++i) { + float v = lb_rgb[i] / 255.0f / scale + e->input_zp; + int q = static_cast(std::lround(v)); + p[i] = static_cast(std::clamp(q, -128, 127)); + } + break; + } + default: + break; // unreachable: rejected at init + } +} + +// Read row i, column c from the output tensor with dtype-aware dequant. +inline float read_out(const engine_impl *e, int i, int c) { + int idx = i * kExpectedOutputCols + c; + switch (e->output_type) { + case kTfLiteFloat32: { + const float *p = e->interpreter->typed_output_tensor(0); + return p[idx]; + } + case kTfLiteUInt8: { + const uint8_t *p = e->interpreter->typed_output_tensor(0); + return (static_cast(p[idx]) - e->output_zp) * e->output_scale; + } + case kTfLiteInt8: { + const int8_t *p = e->interpreter->typed_output_tensor(0); + return (static_cast(p[idx]) - e->output_zp) * e->output_scale; + } + default: + return 0.0f; + } +} + +std::string label_for(const engine_impl *e, int class_id) { + if (class_id >= 0 && static_cast(class_id) < e->labels.size()) { + return e->labels[class_id]; + } + char buf[32]; + snprintf(buf, sizeof buf, "class_%d", class_id); + return std::string(buf); +} + +} // namespace + +#else // !HAVE_LITERT +extern "C" { +#endif + +extern "C" litert_engine_t *litert_engine_acquire(const char *model_path) { +#ifdef HAVE_LITERT + if (!model_path || !*model_path) return nullptr; + if (!g_config.detection_engine.enabled) { + log_error("LiteRT: engine disabled in config ([detection_engine] enabled=false); " + "cannot load %s", model_path); + return nullptr; + } + + std::string canon = canonicalize_path(model_path); + if (canon.empty()) { + log_error("LiteRT: empty/unresolvable model path"); + return nullptr; + } + + std::lock_guard lk(g_registry_mu); + auto it = g_registry.find(canon); + if (it != g_registry.end()) { + it->second->refcount++; + log_info("LiteRT: reused engine for %s (refcount=%d)", + canon.c_str(), it->second->refcount); + return reinterpret_cast(it->second.get()); + } + + auto e = create_engine(canon); + if (!e) return nullptr; + e->refcount = 1; + engine_impl *raw = e.get(); + g_registry.emplace(canon, std::move(e)); + return reinterpret_cast(raw); +#else + (void)model_path; + return NULL; +#endif +} + +extern "C" void litert_engine_release(litert_engine_t *engine) { +#ifdef HAVE_LITERT + if (!engine) return; + auto *e = reinterpret_cast(engine); + std::lock_guard lk(g_registry_mu); + if (--e->refcount > 0) { + log_info("LiteRT: released %s (refcount=%d)", + e->canonical_path.c_str(), e->refcount); + return; + } + log_info("LiteRT: destroying engine for %s", e->canonical_path.c_str()); + g_registry.erase(e->canonical_path); // destroys the unique_ptr +#else + (void)engine; +#endif +} + +extern "C" int litert_engine_input_width(const litert_engine_t *engine) { +#ifdef HAVE_LITERT + if (!engine) return 0; + return reinterpret_cast(engine)->input_w; +#else + (void)engine; + return 0; +#endif +} + +extern "C" int litert_engine_input_height(const litert_engine_t *engine) { +#ifdef HAVE_LITERT + if (!engine) return 0; + return reinterpret_cast(engine)->input_h; +#else + (void)engine; + return 0; +#endif +} + +extern "C" int litert_engine_detect(litert_engine_t *engine, + const uint8_t *rgb, int w, int h, + float per_stream_threshold, + detection_result_t *out) +{ +#ifdef HAVE_LITERT + if (!engine || !rgb || !out || w <= 0 || h <= 0) return -1; + auto *e = reinterpret_cast(engine); + if (!e->interpreter) return -1; + + out->count = 0; + + float threshold = std::clamp(per_stream_threshold, 0.0f, 1.0f); + + std::lock_guard lk(e->inference_mu); + + // Letterbox into a CPU scratch buffer, then convert into the input tensor. + std::vector lb(static_cast(e->input_w) * e->input_h * 3); + LetterboxParams lp; + if (!letterbox_rgb24(rgb, w, h, lb.data(), e->input_w, e->input_h, &lp)) { + return -1; + } + fill_input_tensor(e, lb.data()); + + auto t_start = std::chrono::steady_clock::now(); + if (e->interpreter->Invoke() != kTfLiteOk) { + log_error("LiteRT: Invoke failed for %s", e->canonical_path.c_str()); + return -1; + } + double inference_ms = std::chrono::duration( + std::chrono::steady_clock::now() - t_start).count(); + log_info("LiteRT: inference %.1f ms (%s)", inference_ms, e->canonical_path.c_str()); + + // Heuristic: if any box coordinate exceeds 1.5, assume pixel-space + // (model_w/h). Otherwise treat as normalized 0..1 in model space. + // (Ultralytics TFLite exports vary; this autodetects either.) + float coord_max = 0.0f; + int n = e->output_num_rows; + for (int i = 0; i < n; ++i) { + for (int c = 0; c < 4; ++c) { + coord_max = std::max(coord_max, std::fabs(read_out(e, i, c))); + } + if (coord_max > 1.5f) break; // enough evidence + } + bool in_pixels = coord_max > 1.5f; + + int written = 0; + for (int i = 0; i < n && written < MAX_DETECTIONS; ++i) { + float conf = read_out(e, i, 4); + // NOTE: assumes the end-to-end YOLO head emits rows in descending + // confidence order (the Ultralytics NMS-embedded export convention), + // so the first sub-threshold row means all remaining rows are too. If + // a future model emits unsorted rows this would truncate valid + // detections — switch to `continue` there. + if (conf < threshold) break; // early exit on sorted E2E output + float x1 = read_out(e, i, 0); + float y1 = read_out(e, i, 1); + float x2 = read_out(e, i, 2); + float y2 = read_out(e, i, 3); + int cls = static_cast(std::lround(read_out(e, i, 5))); + + if (!in_pixels) { + x1 *= e->input_w; y1 *= e->input_h; + x2 *= e->input_w; y2 *= e->input_h; + } + + // De-letterbox: subtract pad, divide by scale → original frame px. + x1 = (x1 - lp.pad_x) / lp.scale; + y1 = (y1 - lp.pad_y) / lp.scale; + x2 = (x2 - lp.pad_x) / lp.scale; + y2 = (y2 - lp.pad_y) / lp.scale; + + // Clamp to frame and reorder if needed. + if (x2 < x1) std::swap(x1, x2); + if (y2 < y1) std::swap(y1, y2); + x1 = std::clamp(x1, 0.0f, static_cast(w)); + y1 = std::clamp(y1, 0.0f, static_cast(h)); + x2 = std::clamp(x2, 0.0f, static_cast(w)); + y2 = std::clamp(y2, 0.0f, static_cast(h)); + if (x2 - x1 < 1.0f || y2 - y1 < 1.0f) continue; + + detection_t *d = &out->detections[written]; + d->confidence = conf; + d->x = x1 / w; + d->y = y1 / h; + d->width = (x2 - x1) / w; + d->height = (y2 - y1) / h; + d->track_id = -1; + d->zone_id[0] = '\0'; + std::string lbl = label_for(e, cls); + snprintf(d->label, MAX_LABEL_LENGTH, "%s", lbl.c_str()); + ++written; + } + + out->count = written; + return 0; +#else + (void)engine; (void)rgb; (void)w; (void)h; (void)per_stream_threshold; (void)out; + return -1; +#endif +} + +#ifndef HAVE_LITERT +} // extern "C" +#endif diff --git a/src/video/detection_model.c b/src/video/detection_model.c index d7eade8f..119a1522 100644 --- a/src/video/detection_model.c +++ b/src/video/detection_model.c @@ -25,6 +25,9 @@ #ifdef SOD_ENABLED #include "sod/sod.h" // For sod_cnn_destroy #endif +#ifdef HAVE_LITERT +#include "video/detection/litert_engine.h" +#endif // Static variable to track if we're in shutdown mode static bool in_shutdown_mode = false; @@ -51,14 +54,15 @@ int init_detection_model_system(void) { log_error("Failed to initialize SOD detection system"); } - // Check for TFLite library - void *tflite_handle = dlopen("libtensorflowlite.so", RTLD_LAZY); - if (tflite_handle) { - log_info("TensorFlow Lite library found and loaded"); - dlclose(tflite_handle); - } else { - log_warn("TensorFlow Lite library not found: %s", dlerror()); - } + // Report in-process LiteRT availability. The old dlopen-based probe of + // libtensorflowlite.so was replaced by a vendored LiteRT submodule built + // at configure time when -DENABLE_LITERT=ON. +#ifdef HAVE_LITERT + log_info("In-process LiteRT detection engine compiled in (enabled=%s)", + g_config.detection_engine.enabled ? "true" : "false"); +#else + log_info("In-process LiteRT detection engine NOT compiled in (rebuild with -DENABLE_LITERT=ON to enable)"); +#endif initialized = true; log_info("Detection model system initialized"); @@ -110,14 +114,13 @@ bool is_model_supported(const char *model_path) { return is_sod_available(); } - // Check for TFLite models + // Check for TFLite models (in-process LiteRT engine) if (strcasecmp(ext, ".tflite") == 0) { - void *handle = dlopen("libtensorflowlite.so", RTLD_LAZY); - if (handle) { - dlclose(handle); - return true; - } +#ifdef HAVE_LITERT + return true; +#else return false; +#endif } return false; @@ -169,74 +172,40 @@ const char* get_model_type(const char *model_path) { } /** - * Load a TFLite model + * Load a TFLite model via the in-process LiteRT engine. + * + * Each stream gets its own model_t wrapper (with the per-stream threshold + * recorded on it for use by detect_objects()), but multiple wrappers + * referencing the same .tflite path share one engine via the LiteRT + * registry's refcount. */ static detection_model_t load_tflite_model(const char *model_path, float threshold) { - // Open TFLite library - void *handle = dlopen("libtensorflowlite.so", RTLD_LAZY); - if (!handle) { - log_error("Failed to load TensorFlow Lite library: %s", dlerror()); - return NULL; - } - - // Clear any existing error - dlerror(); - - // Load TFLite functions - void *(*tflite_load_model)(const char *) = dlsym(handle, "tflite_load_model"); - const char *dlsym_error = dlerror(); - if (dlsym_error) { - log_error("Failed to load TFLite function 'tflite_load_model': %s", dlsym_error); - dlclose(handle); +#ifdef HAVE_LITERT + void *engine = litert_engine_acquire(model_path); + if (!engine) { + // litert_engine_acquire already logged the specific cause. return NULL; } - void (*tflite_free_model)(void *) = dlsym(handle, "tflite_free_model"); - dlsym_error = dlerror(); - if (dlsym_error) { - log_error("Failed to load TFLite function 'tflite_free_model': %s", dlsym_error); - dlclose(handle); - return NULL; - } - - void *(*tflite_detect)(void *, const unsigned char *, int, int, int, int *, float) = - dlsym(handle, "tflite_detect"); - dlsym_error = dlerror(); - if (dlsym_error) { - log_error("Failed to load TFLite function 'tflite_detect': %s", dlsym_error); - dlclose(handle); - return NULL; - } - - // Load the model - void *tflite_model = tflite_load_model(model_path); - if (!tflite_model) { - log_error("Failed to load TFLite model: %s", model_path); - dlclose(handle); - return NULL; - } - - // Create model structure model_t *model = (model_t *)malloc(sizeof(model_t)); if (!model) { log_error("Failed to allocate memory for model structure"); - tflite_free_model(tflite_model); - dlclose(handle); + litert_engine_release(engine); return NULL; } + memset(model, 0, sizeof(model_t)); - // Initialize model structure safe_strcpy(model->type, MODEL_TYPE_TFLITE, sizeof(model->type), 0); - model->tflite.handle = handle; - model->tflite.model = tflite_model; - model->tflite.threshold = threshold; - model->tflite.load_model = tflite_load_model; - model->tflite.free_model = tflite_free_model; - model->tflite.detect = tflite_detect; + model->tflite.engine = engine; + model->threshold = threshold; safe_strcpy(model->path, model_path, MAX_PATH_LENGTH, 0); - - log_info("TFLite model loaded: %s", model_path); return model; +#else + (void)threshold; + log_error("LiteRT not compiled in; cannot load .tflite model %s " + "(rebuild with -DENABLE_LITERT=ON)", model_path); + return NULL; +#endif } /** @@ -511,23 +480,15 @@ void unload_detection_model(detection_model_t model) { m->sod_realnet = NULL; } } else if (strcmp(m->type, MODEL_TYPE_TFLITE) == 0) { - // Unload TFLite model - also try during shutdown - if (m->tflite.model && m->tflite.free_model) { - if (is_shutdown_initiated() || in_shutdown_mode) { - log_info("MEMORY LEAK FIX: Destroying TFLite model during shutdown"); - m->tflite.free_model(m->tflite.model); - if (m->tflite.handle) { - dlclose(m->tflite.handle); - } - } else { - m->tflite.free_model(m->tflite.model); - if (m->tflite.handle) { - dlclose(m->tflite.handle); - } - } - m->tflite.model = NULL; - m->tflite.handle = NULL; + // Release reference on the shared LiteRT engine. The engine's + // registry destroys the underlying tflite::Interpreter and delegate + // when the refcount drops to zero. +#ifdef HAVE_LITERT + if (m->tflite.engine) { + litert_engine_release(m->tflite.engine); + m->tflite.engine = NULL; } +#endif } // Log that we're unloading the model diff --git a/src/video/stream_state.c b/src/video/stream_state.c index 31ee2365..a094bfb4 100644 --- a/src/video/stream_state.c +++ b/src/video/stream_state.c @@ -715,6 +715,14 @@ int handle_stream_error(stream_state_manager_t *state, int error_code, const cha // Log the error log_error("Stream '%s' error: %s (code: %d)", state->name, error_message, error_code); + // Persist the human-readable cause for the streams API / UI tooltip. + if (error_message) { + snprintf(state->last_error_message, sizeof state->last_error_message, + "%s", error_message); + } else { + state->last_error_message[0] = '\0'; + } + // Record error in telemetry metrics (classify by error code) { const char *error_type = "io"; @@ -724,15 +732,22 @@ int handle_stream_error(stream_state_manager_t *state, int error_code, const cha error_type = "protocol"; } else if (error_code == -EIO) { error_type = "io"; + } else if (error_code == STREAM_ERR_MODEL_LOAD) { + error_type = "model"; } metrics_record_error(state->name, error_type); } + // Permanent application-level errors that should NOT trigger reconnect: + // they will keep failing on every retry until the operator fixes config. + bool permanent_error = (error_code == STREAM_ERR_MODEL_LOAD); + // Check if we should attempt reconnection bool should_reconnect = false; - // Only attempt reconnection if the stream was active - if (state->state == STREAM_STATE_ACTIVE) { + // Only attempt reconnection if the stream was active AND the error is + // transport-level (not a permanent config-level failure). + if (state->state == STREAM_STATE_ACTIVE && !permanent_error) { should_reconnect = true; // Update state to reconnecting @@ -746,7 +761,7 @@ int handle_stream_error(stream_state_manager_t *state, int error_code, const cha // Record reconnect in telemetry metrics_record_reconnect(state->name); } else { - // If not active, just set to error state + // If not active, or a permanent error, just set to error state. state->state = STREAM_STATE_ERROR; } diff --git a/src/video/unified_detection_thread.c b/src/video/unified_detection_thread.c index 57a07f16..5b2ac184 100644 --- a/src/video/unified_detection_thread.c +++ b/src/video/unified_detection_thread.c @@ -53,6 +53,7 @@ #include "video/mp4_writer_internal.h" #include "video/mp4_recording.h" #include "video/streams.h" +#include "video/stream_state.h" #include "video/go2rtc/go2rtc_stream.h" #include "video/go2rtc/go2rtc_snapshot.h" #include "video/go2rtc/go2rtc_integration.h" @@ -2697,10 +2698,28 @@ static bool detect_on_decoded_frame(unified_detection_ctx_t *ctx, /* Embedded model (SOD / TFLite / etc.). */ if (!ctx->model) { if (ctx->model_path[0] == '\0') return false; + if (ctx->model_load_failed) { + /* One-shot hard error already reported; don't retry. */ + return false; + } ctx->model = load_detection_model(ctx->model_path, ctx->detection_threshold); if (!ctx->model) { - log_warn("[%s] Failed to load detection model: %s", - ctx->stream_name, ctx->model_path); + /* Hard error: drive the stream into STREAM_STATE_ERROR so the + * UI surfaces the cause via streams.error_message. The specific + * reason (missing file, unsupported dtype, etc.) was already + * logged by load_detection_model / the engine. */ + /* Scratch sized exactly for the prefix + longest possible + * model_path (sizeof includes the null). handle_stream_error() + * truncates to STREAM_ERROR_MESSAGE_MAX when persisting. */ + static const char kPrefix[] = "Failed to load detection model: "; + char msg[sizeof kPrefix + MAX_PATH_LENGTH]; + snprintf(msg, sizeof msg, "%s%s", kPrefix, ctx->model_path); + log_error("[%s] %s", ctx->stream_name, msg); + stream_state_manager_t *sm = get_stream_state_by_name(ctx->stream_name); + if (sm) { + handle_stream_error(sm, STREAM_ERR_MODEL_LOAD, msg); + } + ctx->model_load_failed = true; return false; } log_info("[%s] Loaded detection model: %s", ctx->stream_name, ctx->model_path); diff --git a/src/web/api_handlers_detection_models.c b/src/web/api_handlers_detection_models.c index 2eb8bbb0..aac2c379 100644 --- a/src/web/api_handlers_detection_models.c +++ b/src/web/api_handlers_detection_models.c @@ -153,10 +153,14 @@ void handle_get_detection_models(const http_request_t *req, http_response_t *res continue; } - // Check if file is a supported model - bool supported = is_model_supported(full_path); + // Only expose files we can actually run. This hides sidecar files + // (e.g. a model's .labels.txt) and any other unsupported + // file so the model picker lists selectable models only. + if (!is_model_supported(full_path)) { + continue; + } const char *model_type = get_model_type(full_path); - + // Create model object cJSON *model_obj = cJSON_CreateObject(); if (!model_obj) { @@ -169,7 +173,7 @@ void handle_get_detection_models(const http_request_t *req, http_response_t *res cJSON_AddStringToObject(model_obj, "name", entry->d_name); cJSON_AddStringToObject(model_obj, "path", full_path); cJSON_AddStringToObject(model_obj, "type", model_type); - cJSON_AddBoolToObject(model_obj, "supported", supported); + cJSON_AddBoolToObject(model_obj, "supported", true); // unsupported files are filtered out above // Add file size if (stat(full_path, &st) == 0) { diff --git a/src/web/api_handlers_settings.c b/src/web/api_handlers_settings.c index 1484e589..f46faf05 100644 --- a/src/web/api_handlers_settings.c +++ b/src/web/api_handlers_settings.c @@ -466,6 +466,11 @@ void handle_get_settings(const http_request_t *req, http_response_t *res) { cJSON_AddStringToObject(settings, "buffer_strategy", g_config.default_buffer_strategy); cJSON_AddNumberToObject(settings, "detection_grace_period", g_config.detection_grace_period); + // In-process LiteRT detection engine settings + cJSON_AddBoolToObject(settings, "detection_engine_enabled", g_config.detection_engine.enabled); + cJSON_AddNumberToObject(settings, "detection_engine_threads", g_config.detection_engine.num_threads); + cJSON_AddStringToObject(settings, "detection_engine_delegate", g_config.detection_engine.delegate); + // Auth timeout cJSON_AddNumberToObject(settings, "auth_timeout_hours", g_config.auth_timeout_hours); cJSON_AddNumberToObject(settings, "auth_absolute_timeout_hours", g_config.auth_absolute_timeout_hours); @@ -1091,6 +1096,30 @@ void handle_post_settings(const http_request_t *req, http_response_t *res) { log_info("Updated api_detection_backend: %s", g_config.api_detection_backend); } + // In-process LiteRT (TFLite) detection engine settings + cJSON *de_enabled = cJSON_GetObjectItem(settings, "detection_engine_enabled"); + if (de_enabled && cJSON_IsBool(de_enabled)) { + g_config.detection_engine.enabled = cJSON_IsTrue(de_enabled); + settings_changed = true; + log_info("Updated detection_engine.enabled: %s", + g_config.detection_engine.enabled ? "true" : "false"); + } + cJSON *de_threads = cJSON_GetObjectItem(settings, "detection_engine_threads"); + if (de_threads && cJSON_IsNumber(de_threads)) { + config_set_detection_engine_threads(&g_config, de_threads->valueint); + settings_changed = true; + log_info("Updated detection_engine.num_threads: %d", g_config.detection_engine.num_threads); + } + cJSON *de_delegate = cJSON_GetObjectItem(settings, "detection_engine_delegate"); + if (de_delegate && cJSON_IsString(de_delegate)) { + if (config_set_detection_engine_delegate(&g_config, de_delegate->valuestring)) { + settings_changed = true; + log_info("Updated detection_engine.delegate: %s", g_config.detection_engine.delegate); + } else { + log_warn("Ignored unknown detection_engine_delegate '%s'", de_delegate->valuestring); + } + } + // go2rtc settings cJSON *go2rtc_enabled = cJSON_GetObjectItem(settings, "go2rtc_enabled"); if (go2rtc_enabled && cJSON_IsBool(go2rtc_enabled)) { diff --git a/src/web/api_handlers_streams_get.c b/src/web/api_handlers_streams_get.c index 342d831f..2d3d99a7 100644 --- a/src/web/api_handlers_streams_get.c +++ b/src/web/api_handlers_streams_get.c @@ -264,6 +264,13 @@ void handle_get_streams(const http_request_t *req, http_response_t *res) { } cJSON_AddStringToObject(stream_obj, "status", status); + // Surface the specific cause of an Error state (e.g. failed to load + // a detection model) so the UI can show it in a tooltip. + stream_state_manager_t *sm_err = get_stream_state_by_name(db_streams[i].name); + if (sm_err && sm_err->last_error_message[0] != '\0') { + cJSON_AddStringToObject(stream_obj, "error_message", sm_err->last_error_message); + } + // Add go2rtc HLS availability - tells frontend whether go2rtc is providing HLS for this stream cJSON_AddBoolToObject(stream_obj, "go2rtc_hls_available", go2rtc_integration_is_using_go2rtc_for_hls(db_streams[i].name)); @@ -419,6 +426,13 @@ void handle_get_stream(const http_request_t *req, http_response_t *res) { } cJSON_AddStringToObject(stream_obj, "status", status); + // Surface the specific cause of an Error state (e.g. failed to load + // a detection model) so the UI can show it in a tooltip. + stream_state_manager_t *sm_err = get_stream_state_by_name(config.name); + if (sm_err && sm_err->last_error_message[0] != '\0') { + cJSON_AddStringToObject(stream_obj, "error_message", sm_err->last_error_message); + } + // Add go2rtc HLS availability - tells frontend whether go2rtc is providing HLS for this stream cJSON_AddBoolToObject(stream_obj, "go2rtc_hls_available", go2rtc_integration_is_using_go2rtc_for_hls(config.name)); @@ -573,6 +587,12 @@ void handle_get_stream_full(const http_request_t *req, http_response_t *res) { } cJSON_AddStringToObject(stream_obj, "status", status); + // Surface the specific cause of an Error state. + stream_state_manager_t *sm_err2 = get_stream_state_by_name(config.name); + if (sm_err2 && sm_err2->last_error_message[0] != '\0') { + cJSON_AddStringToObject(stream_obj, "error_message", sm_err2->last_error_message); + } + // Add go2rtc HLS availability - tells frontend whether go2rtc is providing HLS for this stream cJSON_AddBoolToObject(stream_obj, "go2rtc_hls_available", go2rtc_integration_is_using_go2rtc_for_hls(config.name)); diff --git a/third_party/litert b/third_party/litert new file mode 160000 index 00000000..9d26e89d --- /dev/null +++ b/third_party/litert @@ -0,0 +1 @@ +Subproject commit 9d26e89d88ef8785b6a1e54ec41ac8add215a125 diff --git a/web/js/components/preact/SettingsView.jsx b/web/js/components/preact/SettingsView.jsx index 693b9e68..83da6e35 100644 --- a/web/js/components/preact/SettingsView.jsx +++ b/web/js/components/preact/SettingsView.jsx @@ -113,6 +113,10 @@ export function SettingsView() { defaultPostBuffer: 10, bufferStrategy: 'auto', detectionGracePeriod: 2, + // In-process LiteRT detection engine + detectionEngineEnabled: false, + detectionEngineThreads: 1, + detectionEngineDelegate: 'xnnpack', // go2rtc settings go2rtcEnabled: true, go2rtcBinaryPath: '/usr/local/bin/go2rtc', @@ -336,6 +340,9 @@ export function SettingsView() { defaultPostBuffer: settingsData.post_detection_buffer ?? 10, bufferStrategy: settingsData.buffer_strategy || 'auto', detectionGracePeriod: settingsData.detection_grace_period ?? 2, + detectionEngineEnabled: settingsData.detection_engine_enabled || false, + detectionEngineThreads: settingsData.detection_engine_threads ?? 1, + detectionEngineDelegate: settingsData.detection_engine_delegate || 'xnnpack', go2rtcEnabled: settingsData.go2rtc_enabled !== undefined ? settingsData.go2rtc_enabled : true, go2rtcBinaryPath: settingsData.go2rtc_binary_path || '/usr/local/bin/go2rtc', go2rtcConfigDir: settingsData.go2rtc_config_dir || '/etc/lightnvr/go2rtc', @@ -451,6 +458,9 @@ export function SettingsView() { post_detection_buffer: parseInt(settings.defaultPostBuffer, 10), buffer_strategy: settings.bufferStrategy, detection_grace_period: parseInt(settings.detectionGracePeriod, 10), + detection_engine_enabled: !!settings.detectionEngineEnabled, + detection_engine_threads: parseInt(settings.detectionEngineThreads, 10) || 1, + detection_engine_delegate: settings.detectionEngineDelegate || 'xnnpack', go2rtc_enabled: settings.go2rtcEnabled, go2rtc_binary_path: settings.go2rtcBinaryPath, go2rtc_config_dir: settings.go2rtcConfigDir, diff --git a/web/js/components/preact/StreamCard.jsx b/web/js/components/preact/StreamCard.jsx index 6d932b03..60036c0f 100644 --- a/web/js/components/preact/StreamCard.jsx +++ b/web/js/components/preact/StreamCard.jsx @@ -250,6 +250,7 @@ export function StreamCard({ color: 'white', opacity: stream.enabled ? 1 : 0.6 }} + title={stream.error_message || undefined} > {statusLabel} diff --git a/web/js/components/preact/settings/DetectionTab.jsx b/web/js/components/preact/settings/DetectionTab.jsx index 18c64833..87a8c42f 100644 --- a/web/js/components/preact/settings/DetectionTab.jsx +++ b/web/js/components/preact/settings/DetectionTab.jsx @@ -110,6 +110,63 @@ export function DetectionTab({ settings, handleInputChange, handleThresholdChang + + {/* In-process LiteRT (TFLite) detection engine — runtime knobs. */} +
+

{t('settings.detectionEngine')}

+
+

+ {t('settings.detectionEngineEnabledHelp')} +

+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
); } diff --git a/web/public/locales/ar.json b/web/public/locales/ar.json index 7fd4b7b4..a5a7753c 100644 --- a/web/public/locales/ar.json +++ b/web/public/locales/ar.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "محرك الاستدلال المستخدم بواسطة API الاكتشاف", + "settings.detectionEngine": "محرك الكشف (LiteRT المحلي)", + "settings.detectionEngineEnabled": "تفعيل المحرك المحلي", + "settings.detectionEngineEnabledHelp": "تشغيل استدلال TFLite/LiteRT داخل LightNVR. التدفقات التي يكون model_path الخاص بها ملف .tflite ستستخدم هذا المحرك.", + "settings.detectionEngineThreads": "Threads", + "settings.detectionEngineDelegate": "Delegate", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (CPU)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "لا شيء", + "streams.errorModelLoad": "فشل تحميل نموذج الكشف", "settings.defaultDetectionThreshold": "عتبة الاكتشاف الافتراضية", "settings.defaultDetectionThresholdHelp": "عتبة الثقة للاكتشاف (0-100%)", "settings.detectionGracePeriod": "فترة السماح للاكتشاف (ثوانٍ)", diff --git a/web/public/locales/de.json b/web/public/locales/de.json index 2f3d8af8..c7a0e47a 100644 --- a/web/public/locales/de.json +++ b/web/public/locales/de.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "Von der Erkennung verwendetes Inferenz-Backend API", + "settings.detectionEngine": "Erkennungs-Engine (lokales LiteRT)", + "settings.detectionEngineEnabled": "Lokale Engine aktivieren", + "settings.detectionEngineEnabledHelp": "TFLite/LiteRT-Inferenz innerhalb von LightNVR ausführen. Streams mit einem .tflite-model_path verwenden diese Engine.", + "settings.detectionEngineThreads": "Threads", + "settings.detectionEngineDelegate": "Delegate", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (CPU-optimiert)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "Keiner (Referenz)", + "streams.errorModelLoad": "Erkennungsmodell konnte nicht geladen werden", "settings.defaultDetectionThreshold": "Standarderkennungsschwelle", "settings.defaultDetectionThresholdHelp": "Vertrauensschwelle für die Erkennung (0–100 %)", "settings.detectionGracePeriod": "Erkennungs-Toleranzzeit (Sekunden)", diff --git a/web/public/locales/en.json b/web/public/locales/en.json index dd761ee1..8a9ab670 100644 --- a/web/public/locales/en.json +++ b/web/public/locales/en.json @@ -1063,6 +1063,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "Inference backend used by the detection API", + "settings.detectionEngine": "Detection Engine (Local LiteRT)", + "settings.detectionEngineEnabled": "Enable Local Engine", + "settings.detectionEngineEnabledHelp": "Run TFLite/LiteRT inference inside LightNVR. Streams whose model_path is a .tflite file will use this engine.", + "settings.detectionEngineThreads": "Threads", + "settings.detectionEngineDelegate": "Delegate", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (CPU optimized)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "None (reference)", + "streams.errorModelLoad": "Detection model failed to load", "settings.defaultDetectionThreshold": "Default Detection Threshold", "settings.defaultDetectionThresholdHelp": "Confidence threshold for detection (0-100%)", "settings.detectionGracePeriod": "Detection Grace Period (seconds)", diff --git a/web/public/locales/es.json b/web/public/locales/es.json index b91ac577..5eecb354 100644 --- a/web/public/locales/es.json +++ b/web/public/locales/es.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "Backend de inferencia utilizado por la detección API", + "settings.detectionEngine": "Motor de detección (LiteRT local)", + "settings.detectionEngineEnabled": "Habilitar motor local", + "settings.detectionEngineEnabledHelp": "Ejecutar inferencia TFLite/LiteRT dentro de LightNVR. Los streams cuyo model_path sea un archivo .tflite usarán este motor.", + "settings.detectionEngineThreads": "Hilos", + "settings.detectionEngineDelegate": "Delegado", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (optimizado para CPU)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "Ninguno (referencia)", + "streams.errorModelLoad": "Error al cargar el modelo de detección", "settings.defaultDetectionThreshold": "Umbral de detección predeterminado", "settings.defaultDetectionThresholdHelp": "Umbral de confianza para la detección (0-100%)", "settings.detectionGracePeriod": "Período de gracia de detección (segundos)", diff --git a/web/public/locales/fr.json b/web/public/locales/fr.json index ce663179..c11954d4 100644 --- a/web/public/locales/fr.json +++ b/web/public/locales/fr.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "Backend d'inférence utilisé par la détection API", + "settings.detectionEngine": "Moteur de détection (LiteRT local)", + "settings.detectionEngineEnabled": "Activer le moteur local", + "settings.detectionEngineEnabledHelp": "Exécute l'inférence TFLite/LiteRT au sein de LightNVR. Les flux dont le model_path est un fichier .tflite utiliseront ce moteur.", + "settings.detectionEngineThreads": "Threads", + "settings.detectionEngineDelegate": "Délégué", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (optimisé CPU)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "Aucun (référence)", + "streams.errorModelLoad": "Échec du chargement du modèle de détection", "settings.defaultDetectionThreshold": "Seuil de détection par défaut", "settings.defaultDetectionThresholdHelp": "Seuil de confiance pour la détection (0-100 %)", "settings.detectionGracePeriod": "Délai de grâce de détection (secondes)", diff --git a/web/public/locales/it.json b/web/public/locales/it.json index 20103cdb..d8826f05 100644 --- a/web/public/locales/it.json +++ b/web/public/locales/it.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "Backend di inferenza utilizzato dal rilevamento API", + "settings.detectionEngine": "Motore di rilevamento (LiteRT locale)", + "settings.detectionEngineEnabled": "Abilita motore locale", + "settings.detectionEngineEnabledHelp": "Esegui inferenza TFLite/LiteRT all'interno di LightNVR. I flussi il cui model_path è un file .tflite useranno questo motore.", + "settings.detectionEngineThreads": "Thread", + "settings.detectionEngineDelegate": "Delegato", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (ottimizzato per CPU)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "Nessuno (riferimento)", + "streams.errorModelLoad": "Caricamento del modello di rilevamento non riuscito", "settings.defaultDetectionThreshold": "Soglia di rilevamento predefinita", "settings.defaultDetectionThresholdHelp": "Soglia di confidenza per il rilevamento (0-100%)", "settings.detectionGracePeriod": "Periodo di tolleranza rilevamento (secondi)", diff --git a/web/public/locales/ja.json b/web/public/locales/ja.json index 320ff2f7..33e223e9 100644 --- a/web/public/locales/ja.json +++ b/web/public/locales/ja.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "検出で使用される推論バックエンド API", + "settings.detectionEngine": "検出エンジン (ローカル LiteRT)", + "settings.detectionEngineEnabled": "ローカルエンジンを有効化", + "settings.detectionEngineEnabledHelp": "LightNVR 内で TFLite/LiteRT 推論を実行します。model_path が .tflite ファイルのストリームはこのエンジンを使用します。", + "settings.detectionEngineThreads": "スレッド数", + "settings.detectionEngineDelegate": "デリゲート", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (CPU 最適化)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "なし (参照)", + "streams.errorModelLoad": "検出モデルの読み込みに失敗しました", "settings.defaultDetectionThreshold": "デフォルトの検出しきい値", "settings.defaultDetectionThresholdHelp": "検出の信頼しきい値 (0 ～ 100%)", "settings.detectionGracePeriod": "検出グレース期間（秒）", diff --git a/web/public/locales/ko.json b/web/public/locales/ko.json index 39e038ad..ca4f9644 100644 --- a/web/public/locales/ko.json +++ b/web/public/locales/ko.json @@ -1063,6 +1063,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "감지 API에 사용되는 추론 백엔드", + "settings.detectionEngine": "감지 엔진 (로컬 LiteRT)", + "settings.detectionEngineEnabled": "로컬 엔진 활성화", + "settings.detectionEngineEnabledHelp": "LightNVR 내부에서 TFLite/LiteRT 추론을 실행합니다. model_path가 .tflite 파일인 스트림은 이 엔진을 사용합니다.", + "settings.detectionEngineThreads": "스레드", + "settings.detectionEngineDelegate": "델리게이트", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (CPU 최적화)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "없음 (참조)", + "streams.errorModelLoad": "감지 모델 로드 실패", "settings.defaultDetectionThreshold": "기본 감지 임계값", "settings.defaultDetectionThresholdHelp": "감지 신뢰도 임계값 (0~100%)", "settings.detectionGracePeriod": "감지 유예 기간 (초)", diff --git a/web/public/locales/nl.json b/web/public/locales/nl.json index c5c39264..c1ead799 100644 --- a/web/public/locales/nl.json +++ b/web/public/locales/nl.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "Inferentie-backend gebruikt door de detectie API", + "settings.detectionEngine": "Detectie-engine (lokaal LiteRT)", + "settings.detectionEngineEnabled": "Lokale engine inschakelen", + "settings.detectionEngineEnabledHelp": "Voer TFLite/LiteRT-inferentie uit binnen LightNVR. Streams waarvan model_path een .tflite-bestand is, gebruiken deze engine.", + "settings.detectionEngineThreads": "Threads", + "settings.detectionEngineDelegate": "Delegate", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (CPU-geoptimaliseerd)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "Geen (referentie)", + "streams.errorModelLoad": "Detectiemodel kon niet worden geladen", "settings.defaultDetectionThreshold": "Standaard detectiedrempel", "settings.defaultDetectionThresholdHelp": "Betrouwbaarheidsdrempel voor detectie (0-100%)", "settings.detectionGracePeriod": "Detectie-uitlooptijd (seconden)", diff --git a/web/public/locales/pl.json b/web/public/locales/pl.json index ed8c43bd..5a7afd12 100644 --- a/web/public/locales/pl.json +++ b/web/public/locales/pl.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "Backend wnioskowania używany przez detekcję API", + "settings.detectionEngine": "Silnik detekcji (lokalny LiteRT)", + "settings.detectionEngineEnabled": "Włącz lokalny silnik", + "settings.detectionEngineEnabledHelp": "Uruchom wnioskowanie TFLite/LiteRT wewnątrz LightNVR. Strumienie, których model_path to plik .tflite, będą używać tego silnika.", + "settings.detectionEngineThreads": "Wątki", + "settings.detectionEngineDelegate": "Delegate", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (zoptymalizowany dla CPU)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "Brak (referencyjny)", + "streams.errorModelLoad": "Nie udało się załadować modelu detekcji", "settings.defaultDetectionThreshold": "Domyślny próg wykrywania", "settings.defaultDetectionThresholdHelp": "Próg ufności dla wykrycia (0-100%)", "settings.detectionGracePeriod": "Okres tolerancji detekcji (sekundy)", diff --git a/web/public/locales/pt-BR.json b/web/public/locales/pt-BR.json index 8eeb7601..2049a1f8 100644 --- a/web/public/locales/pt-BR.json +++ b/web/public/locales/pt-BR.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "Backend de inferência usado pela API de detecção", + "settings.detectionEngine": "Mecanismo de detecção (LiteRT local)", + "settings.detectionEngineEnabled": "Ativar mecanismo local", + "settings.detectionEngineEnabledHelp": "Executa inferência TFLite/LiteRT dentro do LightNVR. Streams cujo model_path é um arquivo .tflite usarão este mecanismo.", + "settings.detectionEngineThreads": "Threads", + "settings.detectionEngineDelegate": "Delegate", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (otimizado para CPU)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "Nenhum (referência)", + "streams.errorModelLoad": "Falha ao carregar o modelo de detecção", "settings.defaultDetectionThreshold": "Limiar de detecção padrão", "settings.defaultDetectionThresholdHelp": "Limiar de confiança para detecção (0-100%)", "settings.detectionGracePeriod": "Período de carência da detecção (segundos)", diff --git a/web/public/locales/pt-PT.json b/web/public/locales/pt-PT.json index 0c29341e..a734b858 100644 --- a/web/public/locales/pt-PT.json +++ b/web/public/locales/pt-PT.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "Backend de inferência usado pela API de deteção", + "settings.detectionEngine": "Motor de deteção (LiteRT local)", + "settings.detectionEngineEnabled": "Ativar motor local", + "settings.detectionEngineEnabledHelp": "Executa inferência TFLite/LiteRT dentro do LightNVR. As streams cujo model_path é um ficheiro .tflite usarão este motor.", + "settings.detectionEngineThreads": "Threads", + "settings.detectionEngineDelegate": "Delegate", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (otimizado para CPU)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "Nenhum (referência)", + "streams.errorModelLoad": "Falha ao carregar o modelo de deteção", "settings.defaultDetectionThreshold": "Limiar de deteção predefinido", "settings.defaultDetectionThresholdHelp": "Limiar de confiança para deteção (0-100%)", "settings.detectionGracePeriod": "Período de tolerância da deteção (segundos)", diff --git a/web/public/locales/ru.json b/web/public/locales/ru.json index 118880e3..38f8bba2 100644 --- a/web/public/locales/ru.json +++ b/web/public/locales/ru.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "ТензорФлоу Лайт", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "Сервер вывода, используемый для обнаружения API", + "settings.detectionEngine": "Локальный движок обнаружения (LiteRT)", + "settings.detectionEngineEnabled": "Включить локальный движок", + "settings.detectionEngineEnabledHelp": "Выполнять вывод TFLite/LiteRT внутри LightNVR. Потоки, model_path которых указывает на .tflite, будут использовать этот движок.", + "settings.detectionEngineThreads": "Потоки", + "settings.detectionEngineDelegate": "Делегат", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (оптимизация CPU)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "Нет (эталон)", + "streams.errorModelLoad": "Не удалось загрузить модель обнаружения", "settings.defaultDetectionThreshold": "Порог обнаружения по умолчанию", "settings.defaultDetectionThresholdHelp": "Порог уверенности для обнаружения (0–100 %)", "settings.detectionGracePeriod": "Льготный период обнаружения (секунды)", diff --git a/web/public/locales/tr.json b/web/public/locales/tr.json index 04a8fdc1..61d1ea55 100644 --- a/web/public/locales/tr.json +++ b/web/public/locales/tr.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "Algılama API tarafından kullanılan çıkarım arka ucu", + "settings.detectionEngine": "Algılama Motoru (Yerel LiteRT)", + "settings.detectionEngineEnabled": "Yerel motoru etkinleştir", + "settings.detectionEngineEnabledHelp": "LightNVR içinde TFLite/LiteRT çıkarımı çalıştır. model_path değeri .tflite olan akışlar bu motoru kullanır.", + "settings.detectionEngineThreads": "İş parçacığı", + "settings.detectionEngineDelegate": "Delege", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (CPU optimize)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "Yok (referans)", + "streams.errorModelLoad": "Algılama modeli yüklenemedi", "settings.defaultDetectionThreshold": "Varsayılan Algılama Eşiği", "settings.defaultDetectionThresholdHelp": "Tespit için güven eşiği (%0-100)", "settings.detectionGracePeriod": "Algılama Tolerans Süresi (saniye)", diff --git a/web/public/locales/uk.json b/web/public/locales/uk.json index ee73ba4f..cf18b3b7 100644 --- a/web/public/locales/uk.json +++ b/web/public/locales/uk.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "Сервер виведення, який використовується виявленням API", + "settings.detectionEngine": "Локальний рушій виявлення (LiteRT)", + "settings.detectionEngineEnabled": "Увімкнути локальний рушій", + "settings.detectionEngineEnabledHelp": "Виконувати інференс TFLite/LiteRT всередині LightNVR. Потоки, model_path яких — файл .tflite, використовуватимуть цей рушій.", + "settings.detectionEngineThreads": "Потоки", + "settings.detectionEngineDelegate": "Делегат", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (оптимізація CPU)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "Немає (еталон)", + "streams.errorModelLoad": "Не вдалося завантажити модель виявлення", "settings.defaultDetectionThreshold": "Поріг виявлення за замовчуванням", "settings.defaultDetectionThresholdHelp": "Достовірний поріг виявлення (0-100%)", "settings.detectionGracePeriod": "Пільговий період виявлення (секунди)", diff --git a/web/public/locales/zh-TW.json b/web/public/locales/zh-TW.json index 16f3139e..bdcfd841 100644 --- a/web/public/locales/zh-TW.json +++ b/web/public/locales/zh-TW.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "檢測所使用的推理後端API", + "settings.detectionEngine": "偵測引擎 (本機 LiteRT)", + "settings.detectionEngineEnabled": "啟用本機引擎", + "settings.detectionEngineEnabledHelp": "在 LightNVR 內執行 TFLite/LiteRT 推論。model_path 為 .tflite 檔案的串流將使用此引擎。", + "settings.detectionEngineThreads": "執行緒", + "settings.detectionEngineDelegate": "委派", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (CPU 最佳化)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "無 (參考)", + "streams.errorModelLoad": "無法載入偵測模型", "settings.defaultDetectionThreshold": "預設檢測閾值", "settings.defaultDetectionThresholdHelp": "檢測置信閾值（0-100%）", "settings.detectionGracePeriod": "檢測寬限期（秒）", diff --git a/web/public/locales/zh.json b/web/public/locales/zh.json index 13944919..e4d43b90 100644 --- a/web/public/locales/zh.json +++ b/web/public/locales/zh.json @@ -1061,6 +1061,15 @@ "settings.apiDetectionBackendTflite": "TensorFlow Lite", "settings.apiDetectionBackendOpencv": "OpenCV DNN", "settings.apiDetectionBackendHelp": "检测使用的推理后端API", + "settings.detectionEngine": "检测引擎 (本地 LiteRT)", + "settings.detectionEngineEnabled": "启用本地引擎", + "settings.detectionEngineEnabledHelp": "在 LightNVR 内运行 TFLite/LiteRT 推理。model_path 为 .tflite 文件的视频流将使用此引擎。", + "settings.detectionEngineThreads": "线程数", + "settings.detectionEngineDelegate": "委托", + "settings.detectionEngineDelegateXnnpack": "XNNPACK (CPU 优化)", + "settings.detectionEngineDelegateGpu": "GPU", + "settings.detectionEngineDelegateNone": "无 (参考)", + "streams.errorModelLoad": "无法加载检测模型", "settings.defaultDetectionThreshold": "默认检测阈值", "settings.defaultDetectionThresholdHelp": "检测置信阈值（0-100%）", "settings.detectionGracePeriod": "检测宽限期（秒）", From 084c44d4125d180016dadf4ef70f19f5a4c6633e Mon Sep 17 00:00:00 2001 From: Piotr Ziecik Date: Sat, 20 Jun 2026 08:21:53 +0200 Subject: [PATCH 09/13] feat(system): report in-process LiteRT detector memory in system stats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The detector-memory stat scanned /proc for an external light-object-detect process; with the in-process LiteRT engine there is none, so it always read 0. - litert_engine tracks a running total of all loaded engines' memory (model flatbuffer + a sum of non-weight tensor buffers, captured at init), maintained on engine create/destroy and exposed via a lock-free litert_engine_registry_memory_bytes() — the reader never touches a live interpreter, so it can't race a multi-threaded Invoke(). - The system API adds that footprint to detectorMemory and subtracts it from the lightnvr process figure, since the engine lives inside this process — otherwise the UI, which sums the two, would double-count it. Co-Authored-By: Claude Opus 4.8 (1M context) --- include/video/detection/litert_engine.h | 7 +++++++ src/video/detection/litert_engine.cc | 27 +++++++++++++++++++++++++ src/web/api_handlers_system.c | 26 ++++++++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/include/video/detection/litert_engine.h b/include/video/detection/litert_engine.h index b3ae08e3..39fc8f01 100644 --- a/include/video/detection/litert_engine.h +++ b/include/video/detection/litert_engine.h @@ -47,6 +47,13 @@ void litert_engine_release(litert_engine_t *engine); int litert_engine_input_width(const litert_engine_t *engine); int litert_engine_input_height(const litert_engine_t *engine); +/* Combined size of the loaded model flatbuffers (weights + graph) across all + * engines — the detector's resident footprint for the system memory stats. + * The tensor arena is deliberately excluded: it can't be measured accurately + * here and over-counting it made the reported figure exceed process RSS. + * Returns 0 when ENABLE_LITERT=OFF or no engines are loaded. Thread-safe. */ +uint64_t litert_engine_registry_memory_bytes(void); + /* Run inference on a single RGB24 frame (packed HxWx3 in original frame * coordinates). The engine letterboxes internally to its model HxW. * diff --git a/src/video/detection/litert_engine.cc b/src/video/detection/litert_engine.cc index 851b4837..9973b5c4 100644 --- a/src/video/detection/litert_engine.cc +++ b/src/video/detection/litert_engine.cc @@ -91,6 +91,8 @@ struct engine_impl { int32_t output_zp = 0; int output_num_rows = 0; // N in [1, N, 6] + size_t mem_bytes = 0; // size of the loaded model flatbuffer (weights+graph), for memory stats + std::vector labels; ~engine_impl() { @@ -105,6 +107,11 @@ struct engine_impl { std::mutex g_registry_mu; std::unordered_map> g_registry; +// Running total of all loaded engines' memory (model flatbuffer + tensor +// arena), maintained on engine create/destroy under g_registry_mu. Lets the +// stats reader be a single lock-free load that never touches a live interpreter. +std::atomic g_engine_memory_bytes{0}; + std::string canonicalize_path(const char *p) { if (!p || !*p) return {}; char buf[PATH_MAX]; @@ -210,6 +217,16 @@ std::unique_ptr create_engine(const std::string &canon_path) { log_error("LiteRT: failed to load model file %s", canon_path.c_str()); return nullptr; } + // Reported as the detector's memory in the system stats. Only the model + // flatbuffer (weights + graph) is measured: it is accurate and always a + // subset of this process's RSS. The interpreter's tensor arena is excluded + // — there is no arena_used_bytes() in this TFLite, and summing tensor sizes + // wildly over-counts it (the planner reuses arena memory across tensors), + // which previously made the subtracted detector figure exceed the lightnvr + // RSS and report 0 for the process. + if (e->model->allocation()) { + e->mem_bytes = e->model->allocation()->bytes(); + } tflite::ops::builtin::BuiltinOpResolver resolver; tflite::InterpreterBuilder builder(*e->model, resolver); @@ -472,6 +489,7 @@ extern "C" litert_engine_t *litert_engine_acquire(const char *model_path) { if (!e) return nullptr; e->refcount = 1; engine_impl *raw = e.get(); + g_engine_memory_bytes.fetch_add(raw->mem_bytes, std::memory_order_relaxed); g_registry.emplace(canon, std::move(e)); return reinterpret_cast(raw); #else @@ -491,6 +509,7 @@ extern "C" void litert_engine_release(litert_engine_t *engine) { return; } log_info("LiteRT: destroying engine for %s", e->canonical_path.c_str()); + g_engine_memory_bytes.fetch_sub(e->mem_bytes, std::memory_order_relaxed); g_registry.erase(e->canonical_path); // destroys the unique_ptr #else (void)engine; @@ -517,6 +536,14 @@ extern "C" int litert_engine_input_height(const litert_engine_t *engine) { #endif } +extern "C" uint64_t litert_engine_registry_memory_bytes(void) { +#ifdef HAVE_LITERT + return g_engine_memory_bytes.load(std::memory_order_relaxed); +#else + return 0; +#endif +} + extern "C" int litert_engine_detect(litert_engine_t *engine, const uint8_t *rgb, int w, int h, float per_stream_threshold, diff --git a/src/web/api_handlers_system.c b/src/web/api_handlers_system.c index c4fb77d4..bfb59705 100644 --- a/src/web/api_handlers_system.c +++ b/src/web/api_handlers_system.c @@ -43,6 +43,9 @@ #include "core/shutdown_coordinator.h" #include "utils/strings.h" #include "video/stream_manager.h" +#ifdef HAVE_LITERT +#include "video/detection/litert_engine.h" +#endif #include "database/db_streams.h" #include "database/db_recordings.h" #include "storage/storage_manager_streams.h" @@ -728,6 +731,15 @@ void handle_get_system_info(const http_request_t *req, http_response_t *res) { unsigned long long system_free = (system_total > system_used) ? (system_total - system_used) : 0; // Get memory information for the LightNVR process + // Compute the in-process LiteRT engine footprint once. It lives inside this + // process, so it is subtracted from the lightnvr process figure below and + // reported separately as detectorMemory — otherwise the UI, which sums the + // process and detector figures, would double-count it. + unsigned long long litert_used = 0; +#ifdef HAVE_LITERT + litert_used = (unsigned long long)litert_engine_registry_memory_bytes(); +#endif + cJSON *memory = cJSON_CreateObject(); unsigned long process_threads = 0; if (memory) { @@ -753,6 +765,11 @@ void handle_get_system_info(const http_request_t *req, http_response_t *res) { // Convert kB to bytes unsigned long long used = vm_rss * 1024; + // The in-process LiteRT detector is reported separately as + // detectorMemory; subtract it here so the process and detector + // figures (summed by the UI) don't double-count the same RSS. + used = (used > litert_used) ? (used - litert_used) : 0; + // Use the system total memory as the total for LightNVR as well // This makes it simpler to understand the memory usage unsigned long long total = system_total; @@ -810,6 +827,15 @@ void handle_get_system_info(const http_request_t *req, http_response_t *res) { log_debug("light-object-detect not running or memory unavailable, using 0"); } + // Add the in-process LiteRT engine footprint (computed above and + // subtracted from the lightnvr process figure, so process + detector + // sum correctly). Detection runs inside this process now, so there is + // no separate detector process to scan for the LiteRT backend. + if (litert_used > 0) { + detector_used += litert_used; + log_debug("in-process LiteRT engine memory: %llu bytes", litert_used); + } + // Use the system total memory as the total for detector as well unsigned long long total = system_total; From 5bd06d5c281540e5a7fa836f1e264cb030580bfd Mon Sep 17 00:00:00 2001 From: Piotr Ziecik Date: Sun, 21 Jun 2026 10:54:48 +0200 Subject: [PATCH 10/13] fix(config): don't reload config on settings save (fixes use-after-free) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handle_post_settings applied every setting to g_config in place and then re-read the config from disk "to ensure changes are applied". That reload ran load_default_config(), which free()s the heap-allocated config.streams array — on the libuv worker thread serving the request, while the main thread is concurrently reading config.streams[i] (e.g. in stop_detection_stream_reader during shutdown/stream teardown). That is a heap use-after-free (confirmed by AddressSanitizer) and matches earlier SIGSEGV cores in main()'s shutdown loop. The reload was redundant: the settings are already in g_config and persisted by save_config(). Drop it. Settings that need more than an in-memory update (max_streams) already set restart_required and take effect on restart. This leaves reload_config() with no callers, so remove it (config.c + config.h). Co-Authored-By: Claude Opus 4.8 (1M context) --- include/core/config.h | 9 ---- src/core/config.c | 85 --------------------------------- src/web/api_handlers_settings.c | 19 ++++---- 3 files changed, 9 insertions(+), 104 deletions(-) diff --git a/include/core/config.h b/include/core/config.h index cb521859..f7d4af41 100644 --- a/include/core/config.h +++ b/include/core/config.h @@ -297,15 +297,6 @@ typedef struct { */ int load_config(config_t *config); -/** - * Reload configuration from disk - * This is used to refresh the global config after settings changes - * - * @param config Pointer to config structure to fill - * @return 0 on success, non-zero on failure - */ -int reload_config(config_t *config); - /** * Save configuration to specified file * diff --git a/src/core/config.c b/src/core/config.c index 4a549d89..c5825d5c 100644 --- a/src/core/config.c +++ b/src/core/config.c @@ -1331,91 +1331,6 @@ int load_config(config_t *config) { return 0; } -/** - * Reload configuration from disk - * This is used to refresh the global config after settings changes - * - * @param config Pointer to config structure to fill - * @return 0 on success, non-zero on failure - */ -int reload_config(config_t *config) { - if (!config) return -1; - - log_info("Reloading configuration from disk"); - - // Save copies of the current config fields needed for comparison - int old_log_level = config->log_level; - int old_web_port = config->web_port; - char old_web_bind_ip[32]; - safe_strcpy(old_web_bind_ip, config->web_bind_ip, sizeof(old_web_bind_ip), 0); - char old_storage_path[MAX_PATH_LENGTH]; - safe_strcpy(old_storage_path, config->storage_path, sizeof(old_storage_path), 0); - char old_storage_path_hls[MAX_PATH_LENGTH]; - safe_strcpy(old_storage_path_hls, config->storage_path_hls, sizeof(old_storage_path_hls), 0); - char old_models_path[MAX_PATH_LENGTH]; - safe_strcpy(old_models_path, config->models_path, sizeof(old_models_path), 0); - uint64_t old_max_storage_size = config->max_storage_size; - int old_retention_days = config->retention_days; - - // Load the configuration - int result = load_config(config); - if (result != 0) { - log_error("Failed to reload configuration"); - return result; - } - - // Log changes - if (old_log_level != config->log_level) { - log_info("Log level changed: %d -> %d", old_log_level, config->log_level); - } - - if (old_web_port != config->web_port) { - log_info("Web port changed: %d -> %d", old_web_port, config->web_port); - log_warn("Web port change requires restart to take effect"); - } - - if (strcmp(old_web_bind_ip, config->web_bind_ip) != 0) { - log_info("Web bind address changed: %s -> %s", old_web_bind_ip, config->web_bind_ip); - log_warn("Web bind address change requires restart to take effect"); - } - - if (strcmp(old_storage_path, config->storage_path) != 0) { - log_info("Storage path changed: %s -> %s", old_storage_path, config->storage_path); - } - - // Log changes to storage_path_hls - if (old_storage_path_hls[0] == '\0' && config->storage_path_hls[0] != '\0') { - log_info("HLS storage path set: %s", config->storage_path_hls); - } else if (old_storage_path_hls[0] != '\0' && config->storage_path_hls[0] == '\0') { - log_info("HLS storage path cleared, will use storage_path"); - } else if (old_storage_path_hls[0] != '\0' && config->storage_path_hls[0] != '\0' && - strcmp(old_storage_path_hls, config->storage_path_hls) != 0) { - log_info("HLS storage path changed: %s -> %s", old_storage_path_hls, config->storage_path_hls); - } - - if (strcmp(old_models_path, config->models_path) != 0) { - log_info("Models path changed: %s -> %s", old_models_path, config->models_path); - } - - if (old_max_storage_size != config->max_storage_size) { - log_info("Max storage size changed: %lu -> %lu bytes", - (unsigned long)old_max_storage_size, - (unsigned long)config->max_storage_size); - } - - if (old_retention_days != config->retention_days) { - log_info("Retention days changed: %d -> %d", old_retention_days, config->retention_days); - } - - // Note: Do not shallow-copy the entire config struct into g_config, as this - // would copy dynamic pointers (e.g., streams) and can lead to dangling - // references or ownership confusion. Assume reload_config operates directly - // on the global configuration instance when called with &g_config. - - log_info("Configuration reloaded successfully"); - return 0; -} - // Save configuration to file in INI format int save_config(const config_t *config, const char *path) { if (!config) { diff --git a/src/web/api_handlers_settings.c b/src/web/api_handlers_settings.c index f46faf05..71d4663d 100644 --- a/src/web/api_handlers_settings.c +++ b/src/web/api_handlers_settings.c @@ -1934,16 +1934,15 @@ void handle_post_settings(const http_request_t *req, http_response_t *res) { log_info("Configuration saved successfully"); - // Reload the configuration to ensure changes are applied - log_info("Reloading configuration after save"); - if (reload_config(&g_config) != 0) { - log_warn("Failed to reload configuration after save, changes may not be applied until restart"); - } else { - log_info("Configuration reloaded successfully"); - - // Verify the database path after reload - log_info("Database path after reload: %s", g_config.db_path); - } + // Do NOT re-read the config from disk here. Every setting above is + // already applied to g_config in place and persisted by + // save_config(), so reloading is redundant — and was dangerous: it + // ran load_default_config(), which free()s the global config.streams + // array on this libuv worker thread while the main thread is reading + // config.streams[...] (a use-after-free; ASan caught it in + // stop_detection_stream_reader). Settings that need more than an + // in-memory update (e.g. max_streams) already set restart_required + // and take effect on the next restart. } else { log_info("No settings changed"); From f78a92662a6e50db37b8e3d7ba90bb89b76ec37c Mon Sep 17 00:00:00 2001 From: Piotr Ziecik Date: Sun, 21 Jun 2026 13:17:05 +0200 Subject: [PATCH 11/13] fix(log): don't print the detection stream name twice The unified detection thread sets log_set_thread_context("Detection", stream_name), so the logger already prefixes "[Detection] [stream]". Many messages also prepend the name manually via "[%s]", producing "[Detection] [stream] [stream] ...". Set the thread context to the component only and let each message's "[%s]" supply the stream name. (Pre-existing; present since before the detection rework.) Co-Authored-By: Claude Opus 4.8 (1M context) --- src/video/unified_detection_thread.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/video/unified_detection_thread.c b/src/video/unified_detection_thread.c index 5b2ac184..2a0fc5a0 100644 --- a/src/video/unified_detection_thread.c +++ b/src/video/unified_detection_thread.c @@ -1580,7 +1580,10 @@ static void *unified_detection_thread_func(void *arg) { char stream_name[MAX_STREAM_NAME]; safe_strcpy(stream_name, ctx->stream_name, sizeof(stream_name), 0); - log_set_thread_context("Detection", stream_name); + // Component only: each message already carries the stream name in its + // own "[%s]" prefix, so also putting it in the thread context would print + // it twice ([Detection] [stream] [stream] ...). + log_set_thread_context("Detection", NULL); log_info("[%s] Unified detection thread started", stream_name); // Silence libav's default stderr logging. Detection streams often From 554ca95cb549f1d852025da826c9398b1bbbab70 Mon Sep 17 00:00:00 2001 From: Piotr Ziecik Date: Tue, 23 Jun 2026 12:57:51 +0200 Subject: [PATCH 12/13] fix(detection): address PR review feedback - CMakeLists: auto-disable LiteRT with a warning instead of FATAL_ERROR when the third_party/litert submodule is absent, so a default configure still builds out of the box; correct the stale "Default OFF" doc comment. - litert_engine.cc: drop the duplicate `extern config_t g_config` declaration -- config.h already declares it with C linkage, so the C++ redeclaration conflicted. - unified_detection_thread: add `rtsps` to the detection-stream protocol whitelist so rtsps:// URLs (accepted by API validation) aren't rejected by FFmpeg. - api_handlers_streams_get: emit `error_message` only when the effective status is "Error", so a stale last_error_message can't linger in a Running/Reconnecting tooltip. - config.c / lightnvr.ini: document that labels come from a sidecar .labels.txt only (embedded-metadata reading is still a TODO); fix the threads default comment (1). - tests: cover detection_url scheme validation -- POST persists an allowed scheme, and both POST and PUT reject a disallowed (file://) scheme with 400. Co-Authored-By: Claude Opus 4.8 (1M context) --- CMakeLists.txt | 22 +++--- config/lightnvr.ini | 4 +- src/core/config.c | 6 +- src/video/detection/litert_engine.cc | 3 +- src/video/unified_detection_thread.c | 2 +- src/web/api_handlers_streams_get.c | 6 +- tests/unit/test_api_handlers_system.c | 100 ++++++++++++++++++++++++++ 7 files changed, 124 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7aeb180d..a96a5957 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,9 +10,10 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) option(ENABLE_SOD "Enable SOD library for object detection" ON) option(SOD_DYNAMIC_LINK "Dynamically link SOD library instead of static linking" OFF) -# In-process LiteRT (TFLite) detection engine. Default OFF — when ON, the -# vendored LiteRT submodule at third_party/litert is built and linked, and -# the litert_engine module is compiled. +# In-process LiteRT (TFLite) detection engine. Default ON, but auto-disabled at +# configure time if the third_party/litert submodule isn't initialized, so a +# fresh checkout still builds. When ON (and present), the vendored LiteRT +# submodule is built and linked, and the litert_engine module is compiled. option(ENABLE_LITERT "Enable in-process LiteRT (TFLite) detection engine" ON) option(LITERT_WITH_XNNPACK "Compile XNNPACK delegate into the LiteRT engine" ON) option(LITERT_WITH_GPU "Compile GPU (OpenCL/GLES) delegate into the LiteRT engine" OFF) @@ -253,15 +254,18 @@ if(ENABLE_SOD) endif() endif() -# Set up LiteRT (in-process TFLite engine) if enabled +# Set up LiteRT (in-process TFLite engine) if enabled. Auto-disable it when the +# submodule isn't initialized so a default configure still succeeds out-of-the-box. set(LITERT_ENGINE_SOURCES "") +if(ENABLE_LITERT AND NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/third_party/litert/tflite/CMakeLists.txt") + message(WARNING + "ENABLE_LITERT is ON but third_party/litert is not initialized; " + "disabling the in-process LiteRT engine. Run " + "'git submodule update --init --recursive third_party/litert' to enable it.") + set(ENABLE_LITERT OFF) +endif() if(ENABLE_LITERT) set(LITERT_SUBDIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/litert/tflite") - if(NOT EXISTS "${LITERT_SUBDIR}/CMakeLists.txt") - message(FATAL_ERROR - "ENABLE_LITERT=ON but ${LITERT_SUBDIR} is missing. " - "Run: git submodule update --init --recursive third_party/litert") - endif() # Forward delegate selections to the LiteRT subproject. set(TFLITE_ENABLE_XNNPACK ${LITERT_WITH_XNNPACK} CACHE BOOL "" FORCE) diff --git a/config/lightnvr.ini b/config/lightnvr.ini index 26ea85ca..29208860 100755 --- a/config/lightnvr.ini +++ b/config/lightnvr.ini @@ -52,8 +52,8 @@ filter_classes = car,motorcycle,truck,bus,bicycle ; Vehicle classes only [detection_engine] ; In-process TFLite/LiteRT inference. When enabled, any stream whose ; model_path ends in .tflite will route through this engine. Requires -; LightNVR built with -DENABLE_LITERT=ON. Labels are read from the model's -; embedded TFLite metadata or a sidecar .labels.txt. +; LightNVR built with -DENABLE_LITERT=ON. Labels are read from a sidecar +; .labels.txt next to the model. enabled = false threads = 1 ; Interpreter threads (1-16); raise to use multiple CPU cores delegate = xnnpack ; xnnpack | gpu | none (gpu requires -DLITERT_WITH_GPU=ON; falls back to CPU otherwise) diff --git a/src/core/config.c b/src/core/config.c index c5825d5c..3b8e9b01 100644 --- a/src/core/config.c +++ b/src/core/config.c @@ -1579,10 +1579,10 @@ int save_config(const config_t *config, const char *path) { // Write in-process LiteRT detection engine settings fprintf(file, "[detection_engine]\n"); fprintf(file, "; In-process TFLite/LiteRT inference. Streams using a .tflite model_path\n"); - fprintf(file, "; will route through this engine. Per-model labels are read from embedded\n"); - fprintf(file, "; TFLite metadata or a sidecar .labels.txt next to the model.\n"); + fprintf(file, "; will route through this engine. Per-model labels are read from a sidecar\n"); + fprintf(file, "; .labels.txt next to the model.\n"); fprintf(file, "enabled = %s\n", config->detection_engine.enabled ? "true" : "false"); - fprintf(file, "threads = %d ; Interpreter threads (1-16, default: 4)\n", + fprintf(file, "threads = %d ; Interpreter threads (1-16, default: 1)\n", config->detection_engine.num_threads); fprintf(file, "delegate = %s ; xnnpack | gpu | none (default: xnnpack; gpu requires -DLITERT_WITH_GPU=ON)\n\n", config->detection_engine.delegate); diff --git a/src/video/detection/litert_engine.cc b/src/video/detection/litert_engine.cc index 9973b5c4..7886f3b7 100644 --- a/src/video/detection/litert_engine.cc +++ b/src/video/detection/litert_engine.cc @@ -61,7 +61,8 @@ extern "C" { #include "tensorflow/lite/delegates/gpu/delegate.h" #endif -extern config_t g_config; +// g_config is declared in core/config.h (included with C linkage above); a +// separate declaration here would conflict on language linkage. namespace { diff --git a/src/video/unified_detection_thread.c b/src/video/unified_detection_thread.c index 2a0fc5a0..f0ff6418 100644 --- a/src/video/unified_detection_thread.c +++ b/src/video/unified_detection_thread.c @@ -682,7 +682,7 @@ static void *detection_stream_thread_func(void *arg) { /* Refuse file://, concat:, subfile: and other local-resource demuxers. * detection_url is user-supplied; lock it to network protocols. */ av_dict_set(&opts, "protocol_whitelist", - "udp,rtp,rtsp,tcp,https,tls,http", 0); + "udp,rtp,rtsp,rtsps,tcp,tls,https,http", 0); int ret = avformat_open_input(&fmt_ctx, ctx->detection_stream_url, NULL, &opts); av_dict_free(&opts); diff --git a/src/web/api_handlers_streams_get.c b/src/web/api_handlers_streams_get.c index 2d3d99a7..37a6b60a 100644 --- a/src/web/api_handlers_streams_get.c +++ b/src/web/api_handlers_streams_get.c @@ -267,7 +267,7 @@ void handle_get_streams(const http_request_t *req, http_response_t *res) { // Surface the specific cause of an Error state (e.g. failed to load // a detection model) so the UI can show it in a tooltip. stream_state_manager_t *sm_err = get_stream_state_by_name(db_streams[i].name); - if (sm_err && sm_err->last_error_message[0] != '\0') { + if (sm_err && sm_err->last_error_message[0] != '\0' && strcmp(status, "Error") == 0) { cJSON_AddStringToObject(stream_obj, "error_message", sm_err->last_error_message); } @@ -429,7 +429,7 @@ void handle_get_stream(const http_request_t *req, http_response_t *res) { // Surface the specific cause of an Error state (e.g. failed to load // a detection model) so the UI can show it in a tooltip. stream_state_manager_t *sm_err = get_stream_state_by_name(config.name); - if (sm_err && sm_err->last_error_message[0] != '\0') { + if (sm_err && sm_err->last_error_message[0] != '\0' && strcmp(status, "Error") == 0) { cJSON_AddStringToObject(stream_obj, "error_message", sm_err->last_error_message); } @@ -589,7 +589,7 @@ void handle_get_stream_full(const http_request_t *req, http_response_t *res) { // Surface the specific cause of an Error state. stream_state_manager_t *sm_err2 = get_stream_state_by_name(config.name); - if (sm_err2 && sm_err2->last_error_message[0] != '\0') { + if (sm_err2 && sm_err2->last_error_message[0] != '\0' && strcmp(status, "Error") == 0) { cJSON_AddStringToObject(stream_obj, "error_message", sm_err2->last_error_message); } diff --git a/tests/unit/test_api_handlers_system.c b/tests/unit/test_api_handlers_system.c index 55283907..56569296 100644 --- a/tests/unit/test_api_handlers_system.c +++ b/tests/unit/test_api_handlers_system.c @@ -386,6 +386,103 @@ void test_handle_put_stream_parses_audio_voice_enhancement(void) { clear_db_streams(); } +/* detection_url validation (api_handlers_streams_modify.c): allowed schemes + * persist, disallowed schemes are rejected with 400 on both POST and PUT. */ +void test_handle_post_stream_persists_detection_url(void) { + clear_db_streams(); + + init_stream_state_manager(16); + init_stream_manager(16); + + http_request_t req; + http_response_t res; + http_request_init(&req); + http_response_init(&res); + + static const char json_body[] = + "{\"name\":\"cam_durl_post\",\"url\":\"rtsp://localhost/stream\"," + "\"detection_url\":\"rtsp://localhost/lowres\"}"; + req.body = (uint8_t *)json_body; + req.body_len = sizeof(json_body) - 1; + + handle_post_stream(&req, &res); + + /* POST persists the config to the DB before attempting to start the stream, + * so detection_url is observable regardless of startup success. */ + stream_config_t got; + TEST_ASSERT_EQUAL_INT(0, get_stream_config_by_name("cam_durl_post", &got)); + TEST_ASSERT_EQUAL_STRING("rtsp://localhost/lowres", got.detection_url); + + usleep(200000); + http_response_free(&res); + shutdown_stream_manager(); + shutdown_stream_state_manager(); + clear_db_streams(); +} + +void test_handle_post_stream_rejects_disallowed_detection_url(void) { + clear_db_streams(); + + init_stream_state_manager(16); + init_stream_manager(16); + + http_request_t req; + http_response_t res; + http_request_init(&req); + http_response_init(&res); + + /* file:// is a local-resource scheme and must be rejected. */ + static const char json_body[] = + "{\"name\":\"cam_durl_bad\",\"url\":\"rtsp://localhost/stream\"," + "\"detection_url\":\"file:///etc/passwd\"}"; + req.body = (uint8_t *)json_body; + req.body_len = sizeof(json_body) - 1; + + handle_post_stream(&req, &res); + + TEST_ASSERT_EQUAL_INT(400, res.status_code); + + /* Rejected before persisting: the stream must not have been created. */ + stream_config_t got; + TEST_ASSERT_NOT_EQUAL(0, get_stream_config_by_name("cam_durl_bad", &got)); + + http_response_free(&res); + shutdown_stream_manager(); + shutdown_stream_state_manager(); + clear_db_streams(); +} + +void test_handle_put_stream_rejects_disallowed_detection_url(void) { + clear_db_streams(); + + stream_config_t s = make_test_stream("cam_durl_put"); + add_stream_config(&s); + + init_stream_state_manager(16); + init_stream_manager(16); + add_stream(&s); + + http_request_t req; + http_response_t res; + http_request_init(&req); + http_response_init(&res); + + safe_strcpy(req.path, "/api/streams/cam_durl_put", sizeof(req.path), 0); + static const char json_body[] = "{\"detection_url\":\"file:///etc/passwd\"}"; + req.body = (uint8_t *)json_body; + req.body_len = sizeof(json_body) - 1; + + handle_put_stream(&req, &res); + + /* PUT validates the scheme synchronously before queuing the async restart. */ + TEST_ASSERT_EQUAL_INT(400, res.status_code); + + http_response_free(&res); + shutdown_stream_manager(); + shutdown_stream_state_manager(); + clear_db_streams(); +} + int main(void) { init_logger(); load_default_config(&g_config); @@ -416,6 +513,9 @@ int main(void) { RUN_TEST(test_handle_get_stream_by_name_includes_audio_voice_enhancement); RUN_TEST(test_handle_post_stream_persists_audio_voice_enhancement); RUN_TEST(test_handle_put_stream_parses_audio_voice_enhancement); + RUN_TEST(test_handle_post_stream_persists_detection_url); + RUN_TEST(test_handle_post_stream_rejects_disallowed_detection_url); + RUN_TEST(test_handle_put_stream_rejects_disallowed_detection_url); int result = UNITY_END(); shutdown_database(); From 7e48f188f97fce7863c50081202c1e5dcbe898ac Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Wed, 24 Jun 2026 00:28:02 -0400 Subject: [PATCH 13/13] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/web/api_handlers_streams_get.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/web/api_handlers_streams_get.c b/src/web/api_handlers_streams_get.c index 37a6b60a..b801283d 100644 --- a/src/web/api_handlers_streams_get.c +++ b/src/web/api_handlers_streams_get.c @@ -267,8 +267,14 @@ void handle_get_streams(const http_request_t *req, http_response_t *res) { // Surface the specific cause of an Error state (e.g. failed to load // a detection model) so the UI can show it in a tooltip. stream_state_manager_t *sm_err = get_stream_state_by_name(db_streams[i].name); - if (sm_err && sm_err->last_error_message[0] != '\0' && strcmp(status, "Error") == 0) { - cJSON_AddStringToObject(stream_obj, "error_message", sm_err->last_error_message); + if (sm_err && strcmp(status, "Error") == 0) { + char err_msg[STREAM_ERROR_MESSAGE_MAX]; + pthread_mutex_lock(&sm_err->mutex); + safe_strcpy(err_msg, sm_err->last_error_message, sizeof(err_msg), 0); + pthread_mutex_unlock(&sm_err->mutex); + if (err_msg[0] != '\0') { + cJSON_AddStringToObject(stream_obj, "error_message", err_msg); + } } // Add go2rtc HLS availability - tells frontend whether go2rtc is providing HLS for this stream