fix(mtmd/qwen3vl): grid selection rewrite + --image-max-tiles override

Rewrite Qwen3VL multi-tile grid selection: enumerate CxR grids that downscale
the image in both dimensions (col*tile_px<=nx, row*tile_px<=ny), exclude 1x1
(equivalent to dyn_size), then pick within a ratio-error tolerance band by max
tile count and min log-ratio error. Fall back to dyn_size when no grid fits. Add
a global overview thumbnail at entries[0].

Add the --image-max-tiles CLI/env override (1..256), applied after load_hparams
so GGUF defaults don't clobber it, and clamp preproc_min_tiles so a low override
can't produce an empty candidate set on InternVL-family models. Guard the
log-ratio division against zero-dimension images.

Hardening:
- Validate clip.vision.preproc_max_tiles read from the GGUF: clamp to [1,256]
  (the --image-max-tiles range) and fall back to 4 with a warning, matching the
  InternVL branch. A crafted/corrupt value could otherwise drive an oversized
  fitting.reserve() and OOM on load or first inference.
- Size fitting.reserve() to max_tiles*ln(max_tiles) (the actual candidate count)
  instead of max_tiles*2, which undershot ~3x at the top of the range.
- Fail a zero-dimension image gracefully (LOG_ERR + return false) instead of
  GGML_ASSERT, so one malformed upload fails its own request rather than
  aborting the whole server. Callers already treat false as a per-request error.
- Doc/log consistency: reflow --image-max-tiles help to the sibling --image-*
  style, add the --image-tile-mode/--image-max-tiles rows to the cli/server
  READMEs, and log preproc_max_tiles as "(custom value)" when overridden.
- Add tests/test-qwen3vl-grid.cpp (test-only hparams constructors bypass
  clip_ctx): max_tiles=1 dyn_size fallback, too-small image, max_tiles=256
  ceiling, and the zero-dimension case.

Author: Rita Law

common/arg.cpp

@@ -2270,6 +2270,17 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             else { throw std::invalid_argument("unknown --image-tile-mode: " + value + " (use batched, sequential, or disabled)"); }
         }
     ).set_examples(mmproj_examples).set_env("LLAMA_ARG_IMAGE_TILE_MODE"));
+    add_opt(common_arg(
+        {"--image-max-tiles"}, "N",
+        "maximum number of tiles for multi-tile vision models (e.g. Qwen3VL), overrides the\n"
+        "value from the GGUF; use when the GGUF lacks the key or the model default is wrong\n"
+        "for your model size (Qwen3VL defaults to 4, too low for 8B+ variants)",
+        [](common_params & params, int value) {
+            if (value < 1)   { throw std::invalid_argument("--image-max-tiles must be >= 1"); }
+            if (value > 256) { throw std::invalid_argument("--image-max-tiles must be <= 256"); }
+            params.image_max_tiles = value;
+        }
+    ).set_examples(mmproj_examples).set_env("LLAMA_ARG_IMAGE_MAX_TILES"));
     if (llama_supports_rpc()) {
         add_opt(common_arg(
             {"--rpc"}, "SERVERS",

common/common.h

@@ -595,6 +595,7 @@ struct common_params {
     int image_min_tokens = -1;
     int image_max_tokens = -1;
     common_image_tile_mode image_tile_mode = COMMON_IMAGE_TILE_MODE_SEQUENTIAL;
+    int image_max_tiles = -1;  // override preproc_max_tiles from GGUF; -1 = use model default
 
     // finetune
     struct lr_opt lr;

tests/CMakeLists.txt

@@ -297,6 +297,12 @@ if (LLAMA_MTMD)
     llama_build_and_test(test-mtmd-c-api.c)
     target_link_libraries(${LLAMA_TEST_NAME} PRIVATE mtmd)
     unset(LLAMA_TEST_NAME)
+
+    set(LLAMA_TEST_NAME test-qwen3vl-grid)
+    llama_build_and_test(test-qwen3vl-grid.cpp)
+    target_link_libraries(${LLAMA_TEST_NAME} PRIVATE mtmd)
+    target_include_directories(${LLAMA_TEST_NAME} PRIVATE ${PROJECT_SOURCE_DIR}/tools/mtmd)
+    unset(LLAMA_TEST_NAME)
 endif()
 
 # GGUF model data fetcher library for tests that need real model metadata

tests/test-qwen3vl-grid.cpp

@@ -0,0 +1,110 @@
+// Unit tests for Qwen3VL multi-tile grid selection (tools/mtmd/mtmd-image.cpp).
+// Exercises mtmd_image_preprocessor_qwen3vl::preprocess() directly via the
+// test-only hparams constructor, without loading a real GGUF model.
+
+#include "clip-model.h"
+#include "mtmd-image.h"
+
+#include <cstdio>
+#include <cstdlib>
+
+// plain assert() is compiled out under -DNDEBUG (this project's default Release
+// build), which would make every check below a silent no-op; use an explicit
+// check that always runs instead.
+#define CHECK(cond) \
+    do { \
+        if (!(cond)) { \
+            fprintf(stderr, "%s:%d: CHECK failed: %s\n", __FILE__, __LINE__, #cond); \
+            abort(); \
+        } \
+    } while (0)
+
+static clip_hparams make_hparams(int image_size, int patch_size) {
+    clip_hparams hp;
+    hp.image_size = image_size;
+    hp.patch_size = patch_size;
+    hp.n_merge = 2; // matches PROJECTOR_TYPE_QWEN3VL default in clip.cpp
+    // dyn_size fallback needs these set; mirrors clip.cpp's set_limit_image_tokens(8, 4096) for Qwen-VL
+    hp.set_limit_image_tokens(8, 4096);
+    return hp;
+}
+
+static clip_image_u8 make_image(int nx, int ny) {
+    clip_image_u8 img;
+    img.nx = nx;
+    img.ny = ny;
+    img.buf.resize((size_t) nx * ny * 3);
+    return img;
+}
+
+// max_tiles == 1 excludes every grid but 1x1, which is itself excluded as
+// "equivalent to dyn_size" -> the candidate set is always empty -> dyn_size fallback.
+static void test_max_tiles_one_falls_back_to_dyn_size() {
+    clip_hparams hp = make_hparams(/*image_size*/ 768, /*patch_size*/ 16);
+    mtmd_image_preprocessor_qwen3vl pre(hp, /*max_tiles*/ 1);
+
+    clip_image_u8 img = make_image(2000, 1000);
+    clip_image_f32_batch out;
+    bool ok = pre.preprocess(img, out);
+
+    CHECK(ok);
+    CHECK(out.grid_x == 0 && out.grid_y == 0);
+    CHECK(out.entries.size() == 1);
+    printf("test_max_tiles_one_falls_back_to_dyn_size: OK\n");
+}
+
+// An image smaller than one tile in both dimensions has no downscaling grid ->
+// empty candidate set -> dyn_size fallback (same outcome as max_tiles=1, different cause).
+static void test_empty_candidate_set_falls_back_to_dyn_size() {
+    clip_hparams hp = make_hparams(/*image_size*/ 768, /*patch_size*/ 16);
+    mtmd_image_preprocessor_qwen3vl pre(hp, /*max_tiles*/ 256);
+
+    clip_image_u8 img = make_image(100, 100);
+    clip_image_f32_batch out;
+    bool ok = pre.preprocess(img, out);
+
+    CHECK(ok);
+    CHECK(out.grid_x == 0 && out.grid_y == 0);
+    CHECK(out.entries.size() == 1);
+    printf("test_empty_candidate_set_falls_back_to_dyn_size: OK\n");
+}
+
+// A large, well-fitting image at the max_tiles=256 ceiling should tile, and the
+// selected grid must respect the tile budget: grid_x*grid_y <= max_tiles.
+static void test_max_tiles_ceiling_selects_valid_grid() {
+    clip_hparams hp = make_hparams(/*image_size*/ 768, /*patch_size*/ 16);
+    mtmd_image_preprocessor_qwen3vl pre(hp, /*max_tiles*/ 256);
+
+    clip_image_u8 img = make_image(768 * 16, 768 * 16); // square, downscales cleanly at many grids
+    clip_image_f32_batch out;
+    bool ok = pre.preprocess(img, out);
+
+    CHECK(ok);
+    CHECK(out.grid_x > 0 && out.grid_y > 0);
+    CHECK(out.grid_x * out.grid_y <= 256);
+    CHECK(out.entries.size() == (size_t)(out.grid_x * out.grid_y + 1)); // +1 overview thumbnail
+    printf("test_max_tiles_ceiling_selects_valid_grid: OK (grid=%dx%d)\n", out.grid_x, out.grid_y);
+}
+
+// A zero-dimension image must fail this one preprocess() call gracefully
+// (return false) rather than aborting the whole process via GGML_ASSERT.
+static void test_zero_dimension_image_fails_gracefully() {
+    clip_hparams hp = make_hparams(/*image_size*/ 768, /*patch_size*/ 16);
+    mtmd_image_preprocessor_qwen3vl pre(hp, /*max_tiles*/ 4);
+
+    clip_image_u8 img = make_image(0, 100);
+    clip_image_f32_batch out;
+    bool ok = pre.preprocess(img, out);
+
+    CHECK(!ok);
+    printf("test_zero_dimension_image_fails_gracefully: OK\n");
+}
+
+int main(void) {
+    test_max_tiles_one_falls_back_to_dyn_size();
+    test_empty_candidate_set_falls_back_to_dyn_size();
+    test_max_tiles_ceiling_selects_valid_grid();
+    test_zero_dimension_image_fails_gracefully();
+    printf("\nAll qwen3vl grid selection tests passed.\n");
+    return 0;
+}

tools/cli/README.md

@@ -165,6 +165,8 @@
 | `--image, --audio FILE` | path to an image or audio file. use with multimodal models, use comma-separated values for multiple files |
 | `--image-min-tokens N` | minimum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MIN_TOKENS) |
 | `--image-max-tokens N` | maximum number of tokens each image can take, only used by vision models with dynamic resolution (default: read from model)<br/>(env: LLAMA_ARG_IMAGE_MAX_TOKENS) |
+| `--image-tile-mode MODE` | tile encoding mode for multi-tile vision models (e.g. Qwen3VL):<br/>  batched    - all tiles in one forward pass<br/>  sequential - tiles encoded one-by-one (default)<br/>  disabled   - tiling disabled, single tile only<br/>(env: LLAMA_ARG_IMAGE_TILE_MODE) |
+| `--image-max-tiles N` | maximum number of tiles for multi-tile vision models (e.g. Qwen3VL), overrides the<br/>value from the GGUF; use when the GGUF lacks the key or the model default is wrong<br/>for your model size (Qwen3VL defaults to 4, too low for 8B+ variants)<br/>(env: LLAMA_ARG_IMAGE_MAX_TILES) |
 | `--chat-template-kwargs STRING` | sets additional params for the json template parser, must be a valid json object string, e.g. '{"key1":"value1","key2":"value2"}'<br/>(env: LLAMA_CHAT_TEMPLATE_KWARGS) |
 | `--jinja, --no-jinja` | whether to use jinja template engine for chat (default: enabled)<br/>(env: LLAMA_ARG_JINJA) |
 | `--reasoning-format FORMAT` | controls whether thought tags are allowed and/or extracted from the response, and in which format they're returned; one of:<br/>- none: leaves thoughts unparsed in `message.content`<br/>- deepseek: puts thoughts in `message.reasoning_content`<br/>- deepseek-legacy: keeps `<think>` tags in `message.content` while also populating `message.reasoning_content`<br/>(default: auto)<br/>(env: LLAMA_ARG_THINK) |

tools/mtmd/clip.cpp

@@ -1416,8 +1416,26 @@ struct clip_model_loader {
                         hparams.image_resize_algo = RESIZE_ALGO_BILINEAR;
                         get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
                         get_u32(KEY_WIN_ATTN_PATTERN, hparams.n_wa_pattern, model.proj_type == PROJECTOR_TYPE_QWEN25VL); // only 2.5 requires it
-                        // optional multi-tile cap; absent in GGUF → stays 0 and the qwen3vl preprocessor falls back to 4
+                        // SigLIP2-Large ViT (n_head=16, n_layer=24) is used for 2B/4B — max_tiles=4 confirmed.
+                        // Larger ViT variants (8B+) use SigLIP2-SO-400M with unknown max_tiles.
+                        // Read from GGUF if present; otherwise default to 4 and warn for unknown ViTs.
+                        hparams.preproc_max_tiles = 4;
+                        const bool has_max_tiles_key = gguf_find_key(ctx_gguf.get(), KEY_PREPROC_MAX_TILES) >= 0;
                         get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false);
+                        if (has_max_tiles_key && (hparams.preproc_max_tiles < 1 || hparams.preproc_max_tiles > 256)) {
+                            LOG_WRN("%s: clip.vision.preproc_max_tiles=%d out of range [1,256] in GGUF — defaulting to 4\n",
+                                    __func__, hparams.preproc_max_tiles);
+                            hparams.preproc_max_tiles = 4;
+                        }
+                        if (!has_max_tiles_key && (hparams.n_head > 16 || hparams.n_layer > 24)) {
+                            const char * model_name =
+                                model.proj_type == PROJECTOR_TYPE_QWEN2VL  ? "Qwen2VL"  :
+                                model.proj_type == PROJECTOR_TYPE_QWEN25VL ? "Qwen2.5VL" : "Qwen3VL";
+                            LOG_WRN("%s: %s large ViT detected (n_head=%d, n_layer=%d) but "
+                                    "clip.vision.preproc_max_tiles not in GGUF — defaulting to 4. "
+                                    "If using an 8B+ model, set the correct value with --image-max-tiles.\n",
+                                    __func__, model_name, hparams.n_head, hparams.n_layer);
+                        }
                         // ref: https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct/blob/main/preprocessor_config.json
                         hparams.set_limit_image_tokens(8, 4096);
                         hparams.warmup_image_size = hparams.image_size; // warmup at actual tile size to match inference graph shape
@@ -2744,6 +2762,18 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
         if (loader.has_vision) {
             ctx_vision = new clip_ctx(ctx_params);
             loader.load_hparams(ctx_vision->model, CLIP_MODALITY_VISION);
+            // apply CLI overrides after load_hparams so GGUF defaults don't clobber them
+            if (ctx_params.image_max_tiles != -1) {
+                ctx_vision->model.hparams.preproc_max_tiles = ctx_params.image_max_tiles;
+                LOG_INF("%s: preproc_max_tiles: %d (custom value)\n", __func__, ctx_vision->model.hparams.preproc_max_tiles);
+                // models with a min-tiles floor (e.g. InternVL) would produce an empty candidate
+                // set if the override drops max below min; clamp min so the invariant holds.
+                if (ctx_vision->model.hparams.preproc_min_tiles > ctx_vision->model.hparams.preproc_max_tiles) {
+                    LOG_WRN("%s: --image-max-tiles=%d is below the model's preproc_min_tiles=%d; clamping min\n",
+                            __func__, ctx_params.image_max_tiles, ctx_vision->model.hparams.preproc_min_tiles);
+                    ctx_vision->model.hparams.preproc_min_tiles = ctx_vision->model.hparams.preproc_max_tiles;
+                }
+            }
             loader.load_tensors(*ctx_vision);
             if (ctx_params.warmup) {
                 loader.warmup(*ctx_vision);

tools/mtmd/clip.h

@@ -51,7 +51,8 @@ struct clip_context_params {
     ggml_backend_sched_eval_callback cb_eval;
     void * cb_eval_user_data;
     const char * backend_device; // optional, if null will use env var or default GPU backend
-    int image_tile_mode; // 0=batched (default), 1=sequential, 2=disabled
+    int image_tile_mode;   // 0=batched (default), 1=sequential, 2=disabled
+    int image_max_tiles;   // override preproc_max_tiles; -1 = use GGUF/model default
 };
 
 struct clip_init_result {

tools/mtmd/mtmd-cli.cpp

@@ -143,7 +143,8 @@ struct mtmd_cli_context {
         mparams.warmup           = params.warmup;
         mparams.image_min_tokens       = params.image_min_tokens;
         mparams.image_max_tokens       = params.image_max_tokens;
-        mparams.image_tile_mode = (int)params.image_tile_mode;
+        mparams.image_tile_mode  = (int)params.image_tile_mode;
+        mparams.image_max_tiles  = params.image_max_tiles;
         if (std::getenv("MTMD_DEBUG_GRAPH") != nullptr) {
             mparams.cb_eval_user_data = &cb_data;
             mparams.cb_eval = common_debug_cb_eval;