Merge pull request #20854 from andriiryzhkov/nr_rawdenoise

[AI] Add RAW denoise (RawNIND, Bayer + Linear) to neural restore module

Author: TurboGit

data/ai_models.json

@@ -35,6 +35,14 @@
       "github_asset": "denoise-nafnet.dtmodel",
       "default": false
     },
+    {
+      "id": "rawdenoise-nind",
+      "name": "raw denoise nind",
+      "description": "UtNet2 raw denoiser trained on RawNIND dataset",
+      "task": "rawdenoise",
+      "github_asset": "rawdenoise-nind.dtmodel",
+      "default": true
+    },
     {
       "id": "upscale-bsrgan",
       "name": "upscale bsrgan",

data/darktableconfig.xml.in

@@ -3772,9 +3772,9 @@
   <dtconfig>
     <name>plugins/lighttable/neural_restore/detail_recovery_bands</name>
     <type>string</type>
-    <default>0.5,0.3,0.1,0.05,0.02</default>
+    <default>0.25,0.15,0.05,0.02,0.01</default>
     <shortdescription>detail recovery wavelet band thresholds</shortdescription>
-    <longdescription>comma-separated sigma multipliers for wavelet detail recovery bands (finest to coarsest). controls how much noise vs texture is recovered by the detail recovery slider</longdescription>
+    <longdescription>comma-separated sigma multipliers for wavelet detail recovery bands (finest to coarsest). controls how much noise vs texture passes through the DWT filter when strength is below 100</longdescription>
   </dtconfig>
   <dtconfig>
     <name>plugins/lighttable/neural_restore/preview_height</name>

dev-doc/AI.md

@@ -17,7 +17,12 @@ src/ai/                          ONNX Runtime backend (darktable_ai static lib)
 
 src/common/ai/                   higher-level AI modules (compiled in lib_darktable)
   segmentation.c/.h                SAM/SegNext interactive masking
-  restore.c/.h                     denoise/upscale tiled inference
+  restore.c/.h                     generic env/ctx lifecycle + model loaders
+  restore_common.h                 private struct defs shared by restore_*
+  restore_rgb.c/.h                 RGB-path denoise + upscale (tiled inference,
+                                   shadow boost, DWT detail recovery)
+  restore_raw_bayer.c/.h           RawNIND Bayer denoise (batch + piped preview)
+  restore_raw_linear.c/.h          RawNIND linear/X-Trans denoise
 
 src/common/ai_models.c/.h       model registry, download, preferences integration
 src/gui/preferences_ai.c        AI preferences tab
@@ -402,6 +407,9 @@ FILE(GLOB SOURCE_FILES_AI
   "common/ai_models.c"
   "common/ai/segmentation.c"
   "common/ai/restore.c"
+  "common/ai/restore_rgb.c"
+  "common/ai/restore_raw_bayer.c"
+  "common/ai/restore_raw_linear.c"
   "common/ai/your_task.c"       # add here
   ...
 )
@@ -455,8 +463,10 @@ dt_your_task_free(ctx);
 | Task | Key | API | Consumer |
 |------|-----|-----|----------|
 | Object Mask | `"mask"` | `src/common/ai/segmentation.h` | `src/develop/masks/object.c` |
-| Denoise | `"denoise"` | `src/common/ai/restore.h` | `src/libs/neural_restore.c` |
-| Upscale | `"upscale"` | `src/common/ai/restore.h` | `src/libs/neural_restore.c` |
+| Denoise | `"denoise"` | `src/common/ai/restore_rgb.h` | `src/libs/neural_restore.c` |
+| Upscale | `"upscale"` | `src/common/ai/restore_rgb.h` | `src/libs/neural_restore.c` |
+| Raw Denoise (Bayer)   | `"rawdenoise"` | `src/common/ai/restore_raw_bayer.h`  | `src/libs/neural_restore.c` |
+| Raw Denoise (Linear)  | `"rawdenoise"` | `src/common/ai/restore_raw_linear.h` | `src/libs/neural_restore.c` |
 
 For model requirements, I/O specifications, tiling strategies, color
 space conventions, ONNX export instructions, and config.json examples

dev-doc/AI_Tasks.md

@@ -131,7 +131,7 @@ repository. Requirements for the decoder export:
 Removes noise from developed images using neural network inference.
 
 **Task key**: `"denoise"`
-**API**: `src/common/ai/restore.h` (`dt_restore_load_denoise`)
+**API**: `src/common/ai/restore.h` (loader: `dt_restore_load_denoise`), `src/common/ai/restore_rgb.h` (processing: `dt_restore_process_tiled`)
 **Consumer**: `src/libs/neural_restore.c`
 
 ### How It Works
@@ -222,7 +222,7 @@ torch.onnx.export(model, dummy_input, "model.onnx",
 Super-resolution upscaling of developed images (2x or 4x).
 
 **Task key**: `"upscale"`
-**API**: `src/common/ai/restore.h` (`dt_restore_load_upscale_x2`, `dt_restore_load_upscale_x4`)
+**API**: `src/common/ai/restore.h` (loaders: `dt_restore_load_upscale_x2`, `dt_restore_load_upscale_x4`), `src/common/ai/restore_rgb.h` (processing: `dt_restore_process_tiled`)
 **Consumer**: `src/libs/neural_restore.c`
 
 ### How It Works

src/CMakeLists.txt

@@ -160,6 +160,7 @@ FILE(GLOB SOURCE_FILES
   "gui/welcome.c"
   "gui/styles_dialog.c"
   "imageio/imageio.c"
+  "imageio/imageio_dng.c"
   "imageio/imageio_jpeg.c"
   "imageio/imageio_module.c"
   "imageio/imageio_pfm.c"
@@ -462,6 +463,9 @@ if(USE_AI)
     "common/ai_models.c"
     "common/ai/segmentation.c"
     "common/ai/restore.c"
+    "common/ai/restore_rgb.c"
+    "common/ai/restore_raw_bayer.c"
+    "common/ai/restore_raw_linear.c"
     "develop/masks/object.c"
     "gui/preferences_ai.c"
   )

src/ai/backend.h

@@ -166,6 +166,14 @@ double dt_ai_model_attribute_double(const dt_ai_model_info_t *info,
 char *dt_ai_model_attribute_string(const dt_ai_model_info_t *info,
                                    const char *key);
 
+/** Return a newly-allocated int array from the JSON-array attribute
+ *  named `key`. *out_count is set to the array length; NULL is returned
+ *  (and *out_count = 0) when the key is absent or not a JSON array.
+ *  Caller frees the returned array with g_free(). */
+int *dt_ai_model_attribute_int_array(const dt_ai_model_info_t *info,
+                                     const char *key,
+                                     int *out_count);
+
 /* --- Discovery --- */
 
 /**
@@ -267,7 +275,8 @@ dt_ai_context_t *dt_ai_load_model_ext(dt_ai_environment_t *env,
                                        dt_ai_provider_t provider,
                                        dt_ai_opt_level_t opt_level,
                                        const dt_ai_dim_override_t *dim_overrides,
-                                       int n_overrides);
+                                       int n_overrides,
+                                       uint32_t ep_flags);
 
 /**
  * @brief Tensor Data Types

src/ai/backend_common.c

@@ -357,7 +357,8 @@ dt_ai_provider_t dt_ai_env_get_provider(dt_ai_environment_t *env)
 extern dt_ai_context_t *
 dt_ai_onnx_load_ext(const char *model_dir, const char *model_file,
                     dt_ai_provider_t provider, dt_ai_opt_level_t opt_level,
-                    const dt_ai_dim_override_t *dim_overrides, int n_overrides);
+                    const dt_ai_dim_override_t *dim_overrides, int n_overrides,
+                    uint32_t ep_flags);
 
 // model loading with backend dispatch
 
@@ -367,7 +368,7 @@ dt_ai_context_t *dt_ai_load_model(dt_ai_environment_t *env,
                                   dt_ai_provider_t provider)
 {
   return dt_ai_load_model_ext(env, model_id, model_file, provider,
-                              DT_AI_OPT_ALL, NULL, 0);
+                              DT_AI_OPT_ALL, NULL, 0, 0);
 }
 
 dt_ai_context_t *dt_ai_load_model_ext(dt_ai_environment_t *env,
@@ -376,7 +377,8 @@ dt_ai_context_t *dt_ai_load_model_ext(dt_ai_environment_t *env,
                                       dt_ai_provider_t provider,
                                       dt_ai_opt_level_t opt_level,
                                       const dt_ai_dim_override_t *dim_overrides,
-                                      int n_overrides)
+                                      int n_overrides,
+                                      uint32_t ep_flags)
 {
   if(!env || !model_id)
     return NULL;
@@ -428,7 +430,7 @@ dt_ai_context_t *dt_ai_load_model_ext(dt_ai_environment_t *env,
   if(strcmp(backend_copy, "onnx") == 0)
   {
     ctx = dt_ai_onnx_load_ext(model_dir, model_file, provider, opt_level,
-                               dim_overrides, n_overrides);
+                               dim_overrides, n_overrides, ep_flags);
   }
   else
   {
@@ -448,6 +450,10 @@ dt_ai_context_t *dt_ai_load_model_ext(dt_ai_environment_t *env,
 // _attribute_node returns the parsed JsonParser plus a borrowed JsonNode*
 // for the named key; caller must g_object_unref the returned parser;
 // returns NULL parser if the attribute set is absent or the key is missing
+//
+// the key accepts a dotted path ("variants.bayer.onnx"): each segment
+// except the last must resolve to a JSON object; the final segment is
+// the leaf lookup and may hold any JSON value type
 static JsonParser *_attribute_node(const dt_ai_model_info_t *info,
                                    const char *key,
                                    JsonNode **out_node)
@@ -467,12 +473,26 @@ static JsonParser *_attribute_node(const dt_ai_model_info_t *info,
     return NULL;
   }
   JsonObject *obj = json_node_get_object(root);
-  if(!json_object_has_member(obj, key))
+  gchar **segments = g_strsplit(key, ".", -1);
+  const int n = g_strv_length(segments);
+  JsonNode *node = NULL;
+  for(int i = 0; i < n; i++)
+  {
+    if(!json_object_has_member(obj, segments[i])) goto out;
+    node = json_object_get_member(obj, segments[i]);
+    if(i == n - 1) break;
+    // intermediate segments must be objects to descend further
+    if(!node || !JSON_NODE_HOLDS_OBJECT(node)) { node = NULL; goto out; }
+    obj = json_node_get_object(node);
+  }
+out:
+  g_strfreev(segments);
+  if(!node)
   {
     g_object_unref(parser);
     return NULL;
   }
-  *out_node = json_object_get_member(obj, key);
+  *out_node = node;
   return parser;
 }
 
@@ -529,6 +549,30 @@ char *dt_ai_model_attribute_string(const dt_ai_model_info_t *info,
   return result;
 }
 
+int *dt_ai_model_attribute_int_array(const dt_ai_model_info_t *info,
+                                     const char *key,
+                                     int *out_count)
+{
+  if(out_count) *out_count = 0;
+  JsonNode *v = NULL;
+  JsonParser *p = _attribute_node(info, key, &v);
+  int *result = NULL;
+  if(v && JSON_NODE_HOLDS_ARRAY(v))
+  {
+    JsonArray *arr = json_node_get_array(v);
+    const guint n = json_array_get_length(arr);
+    if(n > 0)
+    {
+      result = g_new(int, n);
+      for(guint i = 0; i < n; i++)
+        result[i] = (int)json_array_get_int_element(arr, i);
+      if(out_count) *out_count = (int)n;
+    }
+  }
+  if(p) g_object_unref(p);
+  return result;
+}
+
 // provider string conversion
 
 const char *dt_ai_provider_to_string(dt_ai_provider_t provider)

src/ai/backend_onnx.c

@@ -801,7 +801,8 @@ static float _half_to_float(uint16_t h)
 static gboolean _try_provider(OrtSessionOptions *session_opts,
                               const char *symbol_name,
                               const char *provider_name,
-                              const char *device_type)
+                              const char *device_type,
+                              uint32_t flags)
 {
   OrtStatus *status = NULL;
   gboolean ok = FALSE;
@@ -851,7 +852,7 @@ static gboolean _try_provider(OrtSessionOptions *session_opts,
       // integer-argument providers (CUDA, CoreML, DML, MIGraphX, ROCm)
       typedef OrtStatus *(*ProviderAppenderInt)(OrtSessionOptions *, uint32_t);
       ProviderAppenderInt appender = (ProviderAppenderInt)func_ptr;
-      status = appender(session_opts, 0);
+      status = appender(session_opts, flags);
     }
     if(!status)
     {
@@ -880,7 +881,9 @@ static gboolean _try_provider(OrtSessionOptions *session_opts,
 }
 
 static void
-_enable_acceleration(OrtSessionOptions *session_opts, dt_ai_provider_t provider)
+_enable_acceleration(OrtSessionOptions *session_opts,
+                     dt_ai_provider_t provider,
+                     uint32_t coreml_flags)
 {
   switch(provider)
   {
@@ -894,36 +897,36 @@ _enable_acceleration(OrtSessionOptions *session_opts, dt_ai_provider_t provider)
     _try_provider(
       session_opts,
       "OrtSessionOptionsAppendExecutionProvider_CoreML",
-      "Apple CoreML", NULL);
+      "Apple CoreML", NULL, coreml_flags);
 #else
     dt_print(DT_DEBUG_AI, "[darktable_ai] apple CoreML not available on this platform");
 #endif
     break;
 
   case DT_AI_PROVIDER_CUDA:
-    _try_provider(session_opts, "OrtSessionOptionsAppendExecutionProvider_CUDA", "NVIDIA CUDA", NULL);
+    _try_provider(session_opts, "OrtSessionOptionsAppendExecutionProvider_CUDA", "NVIDIA CUDA", NULL, 0);
     break;
 
   case DT_AI_PROVIDER_MIGRAPHX:
     // MIGraphX reads its cache env vars once at provider library
     // load time, so they must be set before CreateEnv() — see
     // _setup_amd_caches() above. OpenVINO (below) takes options
     // per-session, so its cache path is passed inline here
-    if(!_try_provider(session_opts, "OrtSessionOptionsAppendExecutionProvider_MIGraphX", "AMD MIGraphX", NULL))
-      _try_provider(session_opts, "OrtSessionOptionsAppendExecutionProvider_ROCM", "AMD ROCm (legacy)", NULL);
+    if(!_try_provider(session_opts, "OrtSessionOptionsAppendExecutionProvider_MIGraphX", "AMD MIGraphX", NULL, 0))
+      _try_provider(session_opts, "OrtSessionOptionsAppendExecutionProvider_ROCM", "AMD ROCm (legacy)", NULL, 0);
     break;
 
   case DT_AI_PROVIDER_OPENVINO:
     if(!_try_openvino_with_cache(session_opts))
-      _try_provider(session_opts, "OrtSessionOptionsAppendExecutionProvider_OpenVINO", "Intel OpenVINO", "AUTO");
+      _try_provider(session_opts, "OrtSessionOptionsAppendExecutionProvider_OpenVINO", "Intel OpenVINO", "AUTO", 0);
     break;
 
   case DT_AI_PROVIDER_DIRECTML:
 #if defined(_WIN32)
     _try_provider(
       session_opts,
       "OrtSessionOptionsAppendExecutionProvider_DML",
-      "Windows DirectML", NULL);
+      "Windows DirectML", NULL, 0);
 #else
     dt_print(DT_DEBUG_AI, "[darktable_ai] windows DirectML not available on this platform");
 #endif
@@ -936,27 +939,27 @@ _enable_acceleration(OrtSessionOptions *session_opts, dt_ai_provider_t provider)
     _try_provider(
       session_opts,
       "OrtSessionOptionsAppendExecutionProvider_CoreML",
-      "Apple CoreML", NULL);
+      "Apple CoreML", NULL, coreml_flags);
 #elif defined(_WIN32)
     _try_provider(
       session_opts,
       "OrtSessionOptionsAppendExecutionProvider_DML",
-      "Windows DirectML", NULL);
+      "Windows DirectML", NULL, 0);
 #elif defined(__linux__)
     // try CUDA first, then MIGraphX (cache configured at env init)
     if(!_try_provider(
          session_opts,
          "OrtSessionOptionsAppendExecutionProvider_CUDA",
-         "NVIDIA CUDA", NULL))
+         "NVIDIA CUDA", NULL, 0))
     {
       if(!_try_provider(
            session_opts,
            "OrtSessionOptionsAppendExecutionProvider_MIGraphX",
-           "AMD MIGraphX", NULL))
+           "AMD MIGraphX", NULL, 0))
         _try_provider(
           session_opts,
           "OrtSessionOptionsAppendExecutionProvider_ROCM",
-          "AMD ROCm (legacy)", NULL);
+          "AMD ROCm (legacy)", NULL, 0);
     }
 #endif
     break;
@@ -996,20 +999,20 @@ int dt_ai_probe_provider(dt_ai_provider_t provider)
   switch(provider)
   {
   case DT_AI_PROVIDER_COREML:
-    ok = _try_provider(opts, "OrtSessionOptionsAppendExecutionProvider_CoreML", "Apple CoreML", NULL);
+    ok = _try_provider(opts, "OrtSessionOptionsAppendExecutionProvider_CoreML", "Apple CoreML", NULL, 0);
     break;
   case DT_AI_PROVIDER_CUDA:
-    ok = _try_provider(opts, "OrtSessionOptionsAppendExecutionProvider_CUDA", "NVIDIA CUDA", NULL);
+    ok = _try_provider(opts, "OrtSessionOptionsAppendExecutionProvider_CUDA", "NVIDIA CUDA", NULL, 0);
     break;
   case DT_AI_PROVIDER_MIGRAPHX:
-    ok = _try_provider(opts, "OrtSessionOptionsAppendExecutionProvider_MIGraphX", "AMD MIGraphX", NULL)
-      || _try_provider(opts, "OrtSessionOptionsAppendExecutionProvider_ROCM", "AMD ROCm (legacy)", NULL);
+    ok = _try_provider(opts, "OrtSessionOptionsAppendExecutionProvider_MIGraphX", "AMD MIGraphX", NULL, 0)
+      || _try_provider(opts, "OrtSessionOptionsAppendExecutionProvider_ROCM", "AMD ROCm (legacy)", NULL, 0);
     break;
   case DT_AI_PROVIDER_OPENVINO:
-    ok = _try_provider(opts, "OrtSessionOptionsAppendExecutionProvider_OpenVINO", "Intel OpenVINO", "AUTO");
+    ok = _try_provider(opts, "OrtSessionOptionsAppendExecutionProvider_OpenVINO", "Intel OpenVINO", "AUTO", 0);
     break;
   case DT_AI_PROVIDER_DIRECTML:
-    ok = _try_provider(opts, "OrtSessionOptionsAppendExecutionProvider_DML", "Windows DirectML", NULL);
+    ok = _try_provider(opts, "OrtSessionOptionsAppendExecutionProvider_DML", "Windows DirectML", NULL, 0);
     break;
   default:
     break;
@@ -1026,7 +1029,8 @@ int dt_ai_probe_provider(dt_ai_provider_t provider)
 dt_ai_context_t *
 dt_ai_onnx_load_ext(const char *model_dir, const char *model_file,
                     dt_ai_provider_t provider, dt_ai_opt_level_t opt_level,
-                    const dt_ai_dim_override_t *dim_overrides, int n_overrides)
+                    const dt_ai_dim_override_t *dim_overrides, int n_overrides,
+                    uint32_t ep_flags)
 {
   if(!model_dir)
     return NULL;
@@ -1111,7 +1115,7 @@ dt_ai_onnx_load_ext(const char *model_dir, const char *model_file,
   }
 
   // optimize: enable hardware acceleration (AMD caches set at env init)
-  _enable_acceleration(session_opts, provider);
+  _enable_acceleration(session_opts, provider, ep_flags);
 
 #ifdef _WIN32
   // on windows, CreateSession expects a wide character string
@@ -1176,7 +1180,7 @@ dt_ai_onnx_load_ext(const char *model_dir, const char *model_file,
         if(s) g_ort->ReleaseStatus(s);
       }
       if(fallbacks[fb].prov != DT_AI_PROVIDER_CPU)
-        _enable_acceleration(session_opts, fallbacks[fb].prov);
+        _enable_acceleration(session_opts, fallbacks[fb].prov, ep_flags);
 #ifdef _WIN32
       status = g_ort->CreateSession(g_env, onnx_path_wide, session_opts, &ctx->session);
 #else