fix(mtmd/qwen3vl): collapse batched attention loop + sequential default

Replace the per-tile attention loop (a ggml_new_tensor_2d accumulator with
ggml_set_2d writes and per-tile ggml_view_1d position slices, one rope pair
per tile per layer) with zero-copy 3D views over the batched QKV buffer: a
single rope pair runs on the collapsed n_pos*batch_size sequence, then 4D
views feed one block-diagonal build_attn per layer. Fewer graph nodes and
tensor descriptors, scaling with n_layer instead of n_layer*batch_size.

Fix the M-RoPE position layout for batch_size>1: the mrope kernel reads
section s of token i at positions[s*N + i] (N = n_pos*batch_size), so the
buffer must be section-major across the full batched sequence, not tile-
major. The old layout mis-indexed every section beyond the first tile,
encoding (y,y) instead of (y,x). batch_size==1 (the sequential path) is
unchanged. Default --image-tile-mode to sequential.

Author: Rita Law

tools/mtmd/clip.cpp

@@ -166,7 +166,7 @@ struct clip_ctx {
     bool is_allocated = false;
 
     bool debug_output_embeddings = false;
-    clip_image_tile_mode tile_mode = CLIP_IMAGE_TILE_MODE_BATCHED;
+    clip_image_tile_mode tile_mode = CLIP_IMAGE_TILE_MODE_SEQUENTIAL;
 
     // When the GPU backend lacks bf16 support but the GGUF has bf16 weights,
     // we declare the in-context tensors as f16 and convert on disk-load.
@@ -3417,21 +3417,27 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
                 GGML_ASSERT(pw % merge_ratio == 0 && ph % merge_ratio == 0 &&
                             "tile dimensions must be divisible by n_merge");
                 const int n_pos_tile   = pw * ph; // raw patches per tile == n_patches (graph sequence length)
-                // positions layout: tile-major [tile0: y,x,y,x | tile1: y,x,y,x
-                // | ...] each tile's block starts at b * n_pos_tile * 4
-                // (matches ggml_view_1d in graph)
-                std::vector<int32_t> positions((size_t)n_pos_tile * (size_t)batch_size * 4);
+                // positions layout: section-major over the FULL batched sequence. The mrope kernel
+                // reads section s of token i as positions[s * N + i], where N = n_pos_tile * batch_size
+                // is the rope tensor's ne[2]. The four sections are [y, x, y, x]. Every tile gets the
+                // same local-coordinate block (per-tile absolute offsets cancel under relative
+                // attention), so each section just repeats the tile coordinates batch_size times.
+                // For batch_size == 1 this is identical to the old tile-major layout; for
+                // batch_size > 1 (batched mode) it is the only layout the kernel reads correctly —
+                // a tile-major buffer would mis-index every section beyond the first tile.
+                const size_t N = (size_t)n_pos_tile * (size_t)batch_size;
+                std::vector<int32_t> positions(N * 4);
                 for (int b = 0; b < batch_size; b++) {
-                    const size_t base  = (size_t)b * (size_t)n_pos_tile * 4;
+                    const size_t tile_off = (size_t)b * (size_t)n_pos_tile;
                     int ptr = 0;
                     for (int y = 0; y < ph; y += merge_ratio) {
                         for (int x = 0; x < pw; x += merge_ratio) {
                             for (int dy = 0; dy < merge_ratio; dy++) {
                                 for (int dx = 0; dx < merge_ratio; dx++) {
-                                    positions[base + ptr]                          = y + dy;
-                                    positions[base + (size_t)n_pos_tile + ptr]     = x + dx;
-                                    positions[base + (size_t)2 * n_pos_tile + ptr] = y + dy;
-                                    positions[base + (size_t)3 * n_pos_tile + ptr] = x + dx;
+                                    positions[0 * N + tile_off + ptr] = y + dy;
+                                    positions[1 * N + tile_off + ptr] = x + dx;
+                                    positions[2 * N + tile_off + ptr] = y + dy;
+                                    positions[3 * N + tile_off + ptr] = x + dx;
                                     ptr++;
                                 }
                             }

tools/mtmd/clip.h

@@ -51,7 +51,7 @@ struct clip_context_params {
     ggml_backend_sched_eval_callback cb_eval;
     void * cb_eval_user_data;
     const char * backend_device; // optional, if null will use env var or default GPU backend
-    int image_tile_mode;   // 0=batched (default), 1=sequential, 2=disabled
+    int image_tile_mode;   // 0=batched, 1=sequential (default), 2=disabled
     int image_max_tiles;   // override preproc_max_tiles; -1 = use GGUF/model default
 };
 

tools/mtmd/debug/mtmd-debug.cpp

@@ -88,6 +88,8 @@ int main(int argc, char ** argv) {
         mparams.warmup           = params.warmup;
         mparams.image_min_tokens = params.image_min_tokens;
         mparams.image_max_tokens = params.image_max_tokens;
+        mparams.image_tile_mode  = (int) params.image_tile_mode;
+        mparams.image_max_tiles  = params.image_max_tiles;
         {
             // always enable debug callback
             mparams.cb_eval_user_data = &cb_data;

tools/mtmd/models/qwen3vl.cpp

@@ -9,9 +9,9 @@ ggml_cgraph * clip_graph_qwen3vl::build() {
     // QKV/FFN run fully batched; attention is per-tile (each tile attends only to its own tokens).
     // M-RoPE offsets are mathematically inert in the vision encoder: relative attention cancels
     // any per-tile absolute offset. Tile arrangement reaches the LM via decoder positions in mtmd.cpp.
-    const int n_pos            = n_patches;         // patches per tile
-    const int n_pos_total      = n_pos * batch_size; // total sequence length (all tiles)
-    const int num_position_ids = n_pos_total * 4;   // M-RoPE: 4 coords per patch
+    const int     n_pos            = n_patches;                        // patches per tile
+    const int64_t n_pos_total      = (int64_t)n_pos * batch_size;      // total sequence length (all tiles)
+    const int64_t num_position_ids = n_pos_total * 4;                  // M-RoPE: 4 coords per patch
 
     norm_type norm_t = NORM_TYPE_NORMAL;
 
@@ -120,36 +120,37 @@ ggml_cgraph * clip_graph_qwen3vl::build() {
             cb(Kcur, "Kcur", il);
             cb(Vcur, "Vcur", il);
 
-            // Per-tile attention: each tile attends only to its own n_pos tokens.
-            // QKV/FFN linear layers above are fully batched and run in parallel.
-            ggml_tensor * attn_out = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_pos * batch_size);
-            for (int b = 0; b < batch_size; b++) {
-                ggml_tensor * Q_b = ggml_view_3d(ctx0, Qcur, d_head, n_head, n_pos,
-                    Qcur->nb[1], Qcur->nb[2], (size_t)b * Qcur->nb[3]);
-                ggml_tensor * K_b = ggml_view_3d(ctx0, Kcur, d_head, n_head, n_pos,
-                    Kcur->nb[1], Kcur->nb[2], (size_t)b * Kcur->nb[3]);
-                ggml_tensor * V_b = ggml_view_3d(ctx0, Vcur, d_head, n_head, n_pos,
-                    Vcur->nb[1], Vcur->nb[2], (size_t)b * Vcur->nb[3]);
-
-                ggml_tensor * pos_b = ggml_view_1d(ctx0, positions, n_pos * 4,
-                    (size_t)b * n_pos * 4 * sizeof(int32_t));
-
-                Q_b = ggml_rope_multi(ctx0, Q_b, pos_b, nullptr,
-                    d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);
-                K_b = ggml_rope_multi(ctx0, K_b, pos_b, nullptr,
-                    d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);
-
-                cb(Q_b, "Qcur_rope", il);
-                cb(K_b, "Kcur_rope", il);
-
-                ggml_tensor * out_b = build_attn(layer.o_w, layer.o_b,
-                    Q_b, K_b, V_b, nullptr, kq_scale, il);
-
-                attn_out = ggml_set_2d(ctx0, attn_out,
-                    ggml_reshape_2d(ctx0, out_b, n_embd, n_pos),
-                    n_embd * sizeof(float),
-                    (size_t)b * n_embd * n_pos * sizeof(float));
-            }
+            // Batched block-diagonal attention: each tile attends only to its own n_pos tokens.
+            // nb[3] = nb[2] * n_pos on the 4D views, so tiles are contiguous in sequence space.
+            // Create zero-copy 3D non-contiguous views [d_head, n_head, n_pos*batch_size] with
+            // stride nb[2] between positions — same stride the old per-tile loop used per tile.
+            // rope assert: ne[2]*4 == positions.ne[0] → n_pos*batch_size*4 == num_position_ids ✅
+            // After rope, rebuild 4D via ggml_view_4d (no copy). ggml_mul_mat in build_attn
+            // treats ne[3] as batch dim, computing an independent n_pos×n_pos attention per tile.
+            // NOTE: verify on Metal, Vulkan (Android), and OpenCL that ggml_mul_mat correctly
+            // iterates ne[3] for both KQ and KQV products before merging.
+            ggml_tensor * Q3 = ggml_view_3d(ctx0, Qcur, d_head, n_head, n_pos * batch_size,
+                Qcur->nb[1], Qcur->nb[2], 0);
+            ggml_tensor * K3 = ggml_view_3d(ctx0, Kcur, d_head, n_head, n_pos * batch_size,
+                Kcur->nb[1], Kcur->nb[2], 0);
+
+            Q3 = ggml_rope_multi(ctx0, Q3, positions, nullptr,
+                d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);
+            K3 = ggml_rope_multi(ctx0, K3, positions, nullptr,
+                d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);
+
+            // Rebuild 4D view from rope output (same non-contiguous stride layout as Qcur/Kcur)
+            ggml_tensor * Q4 = ggml_view_4d(ctx0, Q3, d_head, n_head, n_pos, batch_size,
+                Q3->nb[1], Q3->nb[2], Q3->nb[2] * n_pos, 0);
+            ggml_tensor * K4 = ggml_view_4d(ctx0, K3, d_head, n_head, n_pos, batch_size,
+                K3->nb[1], K3->nb[2], K3->nb[2] * n_pos, 0);
+
+            cb(Q4, "Qcur_rope", il);
+            cb(K4, "Kcur_rope", il);
+
+            // build_attn output: [n_embd, n_pos * batch_size]
+            ggml_tensor * attn_out = build_attn(layer.o_w, layer.o_b,
+                Q4, K4, Vcur, nullptr, kq_scale, il);
             cb(attn_out, "attn_out", il);
 
             // [n_embd, n_pos * batch_size] → [n_embd, n_pos, batch_size]

tools/mtmd/mtmd-cli.cpp

@@ -141,8 +141,8 @@ struct mtmd_cli_context {
         mparams.n_threads        = params.cpuparams.n_threads;
         mparams.flash_attn_type  = params.flash_attn_type;
         mparams.warmup           = params.warmup;
-        mparams.image_min_tokens       = params.image_min_tokens;
-        mparams.image_max_tokens       = params.image_max_tokens;
+        mparams.image_min_tokens = params.image_min_tokens;
+        mparams.image_max_tokens = params.image_max_tokens;
         mparams.image_tile_mode  = (int)params.image_tile_mode;
         mparams.image_max_tiles  = params.image_max_tiles;
         if (std::getenv("MTMD_DEBUG_GRAPH") != nullptr) {

tools/mtmd/mtmd.cpp

@@ -615,6 +615,8 @@ struct mtmd_caps mtmd_get_cap_from_file(const char * fname) {
         cp.cb_eval          = nullptr;
         cp.cb_eval_user_data= nullptr;
         cp.backend_device   = nullptr;
+        cp.image_tile_mode  = CLIP_IMAGE_TILE_MODE_SEQUENTIAL;
+        cp.image_max_tiles  = -1; // -1 = use model default
         clip_init_result init = clip_init(fname, cp);
         if (init.ctx_v != nullptr) {
             cap.inp_vision = clip_has_vision_encoder(init.ctx_v);
@@ -649,6 +651,8 @@ std::map<ggml_backend_dev_t, size_t> mtmd_get_memory_usage(
         cp.cb_eval          = nullptr;
         cp.cb_eval_user_data= nullptr;
         cp.backend_device   = nullptr;
+        cp.image_tile_mode  = ctx_params.image_tile_mode;
+        cp.image_max_tiles  = ctx_params.image_max_tiles;
         clip_init_result init = clip_init(mmproj_fname, cp);
         auto merge = [&](struct clip_ctx * c) {
             if (c == nullptr) return;

tools/mtmd/mtmd.h

@@ -100,7 +100,7 @@ struct mtmd_context_params {
 
     const char * backend_device; // optional GPU backend name (e.g. "CUDA", "Metal", "Vulkan"), if null will use env var or default
 
-    // tile encoding mode for multi-tile vision models (Qwen3VL): 0=batched (default), 1=sequential, 2=disabled
+    // tile encoding mode for multi-tile vision models (Qwen3VL): 0=batched, 1=sequential (default), 2=disabled
     int image_tile_mode;
 
     // override preproc_max_tiles from GGUF; -1 = use model default (4 for Qwen3VL 2B/4B)