fix(mtmd/qwen3vl): validate image_max_tiles at library boundary, scope to Qwen3VL

Addresses review on PR #175:

- Clamp the preproc_max_tiles override to [1,256] in clip_init. Bindings populate
  clip_context_params directly and bypass the CLI's range check; an unbounded value
  reached the O(max_tiles*log max_tiles) grid-fitting reserve in mtmd-image.cpp and
  could request hundreds of GB -> std::bad_alloc -> process crash.
- Extract a shared CLIP_PREPROC_MAX_TILES_LIMIT (clip-impl.h) and use it at both the
  GGUF-read and override sites so the bound can't drift.
- Scope the override to PROJECTOR_TYPE_QWEN3VL (the only preprocessor that reads the
  field live); warn and ignore for other projectors. Drop the dead InternVL min-tiles
  clamp (InternVL reads a candidate list frozen in load_hparams, not the live field).
- Gate the "set --image-max-tiles" suggestion in the shared Qwen2/2.5/3 case to
  Qwen3VL only; the flag is inert for Qwen2/2.5VL (dyn_size).

Author: Rita Law

tools/mtmd/clip-impl.h

@@ -17,6 +17,11 @@
 
 #define MTMD_INTERNAL_HEADER
 
+// Upper bound on preproc_max_tiles, enforced at every site that sets it (GGUF read,
+// CLI/binding override). A larger value would flow into the O(max_tiles·log max_tiles)
+// grid-fitting reserve in mtmd-image.cpp and can request hundreds of GB -> std::bad_alloc.
+constexpr int CLIP_PREPROC_MAX_TILES_LIMIT = 256;
+
 #define KEY_FTYPE               "general.file_type"
 #define KEY_NAME                "general.name"
 #define KEY_DESCRIPTION         "general.description"

tools/mtmd/clip.cpp

@@ -1422,19 +1422,27 @@ struct clip_model_loader {
                         hparams.preproc_max_tiles = 4;
                         const bool has_max_tiles_key = gguf_find_key(ctx_gguf.get(), KEY_PREPROC_MAX_TILES) >= 0;
                         get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false);
-                        if (has_max_tiles_key && (hparams.preproc_max_tiles < 1 || hparams.preproc_max_tiles > 256)) {
-                            LOG_WRN("%s: clip.vision.preproc_max_tiles=%d out of range [1,256] in GGUF — defaulting to 4\n",
-                                    __func__, hparams.preproc_max_tiles);
+                        if (has_max_tiles_key && (hparams.preproc_max_tiles < 1 || hparams.preproc_max_tiles > CLIP_PREPROC_MAX_TILES_LIMIT)) {
+                            LOG_WRN("%s: clip.vision.preproc_max_tiles=%d out of range [1,%d] in GGUF — defaulting to 4\n",
+                                    __func__, hparams.preproc_max_tiles, CLIP_PREPROC_MAX_TILES_LIMIT);
                             hparams.preproc_max_tiles = 4;
                         }
                         if (!has_max_tiles_key && (hparams.n_head > 16 || hparams.n_layer > 24)) {
                             const char * model_name =
                                 model.proj_type == PROJECTOR_TYPE_QWEN2VL  ? "Qwen2VL"  :
                                 model.proj_type == PROJECTOR_TYPE_QWEN25VL ? "Qwen2.5VL" : "Qwen3VL";
-                            LOG_WRN("%s: %s large ViT detected (n_head=%d, n_layer=%d) but "
-                                    "clip.vision.preproc_max_tiles not in GGUF — defaulting to 4. "
-                                    "If using an 8B+ model, set the correct value with --image-max-tiles.\n",
-                                    __func__, model_name, hparams.n_head, hparams.n_layer);
+                            // preproc_max_tiles is only read by the Qwen3VL preprocessor; Qwen2/2.5VL
+                            // use dyn_size, so --image-max-tiles is inert there and must not be suggested.
+                            if (model.proj_type == PROJECTOR_TYPE_QWEN3VL) {
+                                LOG_WRN("%s: %s large ViT detected (n_head=%d, n_layer=%d) but "
+                                        "clip.vision.preproc_max_tiles not in GGUF — defaulting to 4. "
+                                        "If using an 8B+ model, set the correct value with --image-max-tiles.\n",
+                                        __func__, model_name, hparams.n_head, hparams.n_layer);
+                            } else {
+                                LOG_WRN("%s: %s large ViT detected (n_head=%d, n_layer=%d) but "
+                                        "clip.vision.preproc_max_tiles not in GGUF — defaulting to 4.\n",
+                                        __func__, model_name, hparams.n_head, hparams.n_layer);
+                            }
                         }
                         // ref: https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct/blob/main/preprocessor_config.json
                         hparams.set_limit_image_tokens(8, 4096);
@@ -2762,16 +2770,25 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
         if (loader.has_vision) {
             ctx_vision = new clip_ctx(ctx_params);
             loader.load_hparams(ctx_vision->model, CLIP_MODALITY_VISION);
-            // apply CLI overrides after load_hparams so GGUF defaults don't clobber them
+            // apply CLI/binding overrides after load_hparams so GGUF defaults don't clobber them.
+            // Only the Qwen3VL preprocessor reads preproc_max_tiles live; Qwen2/2.5VL use dyn_size
+            // and InternVL uses a candidate list already frozen in load_hparams, so the override is
+            // inert for them — apply (and validate) it only where it takes effect.
             if (ctx_params.image_max_tiles != -1) {
-                ctx_vision->model.hparams.preproc_max_tiles = ctx_params.image_max_tiles;
-                LOG_INF("%s: preproc_max_tiles: %d (custom value)\n", __func__, ctx_vision->model.hparams.preproc_max_tiles);
-                // models with a min-tiles floor (e.g. InternVL) would produce an empty candidate
-                // set if the override drops max below min; clamp min so the invariant holds.
-                if (ctx_vision->model.hparams.preproc_min_tiles > ctx_vision->model.hparams.preproc_max_tiles) {
-                    LOG_WRN("%s: --image-max-tiles=%d is below the model's preproc_min_tiles=%d; clamping min\n",
-                            __func__, ctx_params.image_max_tiles, ctx_vision->model.hparams.preproc_min_tiles);
-                    ctx_vision->model.hparams.preproc_min_tiles = ctx_vision->model.hparams.preproc_max_tiles;
+                if (ctx_vision->model.proj_type != PROJECTOR_TYPE_QWEN3VL) {
+                    LOG_WRN("%s: --image-max-tiles is only supported for Qwen3VL; ignoring for this model\n", __func__);
+                } else {
+                    int max_tiles = ctx_params.image_max_tiles;
+                    // Re-validate at this library boundary: bindings populate clip_context_params
+                    // directly and bypass the CLI's [1,256] check. An unbounded value reaches the
+                    // grid-fitting reserve in mtmd-image.cpp and can OOM the process.
+                    if (max_tiles < 1 || max_tiles > CLIP_PREPROC_MAX_TILES_LIMIT) {
+                        LOG_WRN("%s: --image-max-tiles=%d out of range [1,%d]; clamping\n",
+                                __func__, max_tiles, CLIP_PREPROC_MAX_TILES_LIMIT);
+                        max_tiles = std::min(std::max(max_tiles, 1), CLIP_PREPROC_MAX_TILES_LIMIT);
+                    }
+                    ctx_vision->model.hparams.preproc_max_tiles = max_tiles;
+                    LOG_INF("%s: preproc_max_tiles: %d (custom value)\n", __func__, max_tiles);
                 }
             }
             loader.load_tensors(*ctx_vision);