QVAC-21914 fix: clip FA AUTO on non-coopmat GPUs + ggml-opencl submission hardening (Pixel 9 OOM, S25 Adreno abort)

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -18,6 +18,8 @@
 #include <inttypes.h>
 #include <string.h>
 
+#include <algorithm>
+#include <cerrno>
 #include <cstddef>
 #include <cstdint>
 #include <fstream>
@@ -404,6 +406,13 @@ struct ggml_backend_opencl_context {
     bool has_vector_subgroup_broadcast;
     bool disable_fusion;
 
+    // QVAC-21914 submission bounds (resolved once at init from env, logged there).
+    // flush_work_budget: bytes of estimated enqueued work per driver submission in
+    // graph_compute; 0 disables. fa_max_nq: max q rows per flash-attention dispatch;
+    // 0 disables chunking.
+    int64_t flush_work_budget;
+    int     fa_max_nq;
+
     bool adreno_has_large_buffer;
     bool adreno_use_large_buffer;
     ggml_cl_compiler_version adreno_cl_compiler_version;
@@ -3654,6 +3663,36 @@ static ggml_backend_opencl_context * ggml_cl2_init(ggml_backend_dev_t dev) {
 
     backend_ctx->disable_fusion = getenv("GGML_OPENCL_DISABLE_FUSION") != nullptr;
 
+    // QVAC-21914 submission bounds. Robust parse (strtol, clamp, warn on garbage
+    // instead of silently disabling the mitigation) and log the effective values,
+    // matching the file's other env knobs.
+    auto parse_env_i64 = [](const char * name, int64_t defval, int64_t maxval) -> int64_t {
+        const char * env = std::getenv(name);
+        if (!env || !env[0]) {
+            return defval;
+        }
+        errno = 0;
+        char * end = nullptr;
+        long long v = std::strtoll(env, &end, 10);
+        if (errno != 0 || end == env || *end != '\0' || v < 0) {
+            GGML_LOG_WARN("ggml_opencl: invalid %s='%s', using default %lld\n",
+                          name, env, (long long) defval);
+            return defval;
+        }
+        if (v > maxval) {
+            GGML_LOG_WARN("ggml_opencl: %s=%lld exceeds max, clamping to %lld\n",
+                          name, v, (long long) maxval);
+            return maxval;
+        }
+        return (int64_t) v;
+    };
+    backend_ctx->flush_work_budget = parse_env_i64("GGML_OPENCL_FLUSH_WORK_MB", 512, INT64_MAX >> 20) * (1ll << 20);
+    backend_ctx->fa_max_nq         = (int) parse_env_i64("GGML_OPENCL_FA_MAX_NQ", 4096, INT32_MAX);
+    GGML_LOG_INFO("ggml_opencl: flush work budget: %lld MB (0 = disabled)\n",
+                  (long long) (backend_ctx->flush_work_budget >> 20));
+    GGML_LOG_INFO("ggml_opencl: flash attention max q rows per dispatch: %d (0 = disabled)\n",
+                  backend_ctx->fa_max_nq);
+
     dev_ctx->backend_ctx = backend_ctx.release();
     return dev_ctx->backend_ctx;
 }
@@ -4231,9 +4270,48 @@ static void ggml_opencl_op_rms_norm_fused(ggml_backend_t backend, ggml_tensor *
 static void ggml_opencl_op_norm_fused(ggml_backend_t backend, ggml_tensor * norm_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor);
 static void ggml_opencl_op_group_norm_fused(ggml_backend_t backend, ggml_tensor * gn_tensor, ggml_tensor * mul_tensor, ggml_tensor * add_tensor);
 
+// QVAC-21914: cheap per-node proxy for enqueued GPU work, used to bound the
+// driver's per-submission batch in graph_compute. Precision is irrelevant —
+// only the ~3 orders of magnitude between a per-token decode step (~10-50 ms
+// of GPU work) and a monolithic 16k-patch ViT encode (~48 s) must separate,
+// and they do: reduction-heavy ops scale by how often their inputs are re-read.
+static int64_t ggml_opencl_node_work_estimate(const ggml_tensor * node) {
+    switch (node->op) {
+        case GGML_OP_MUL_MAT:
+        case GGML_OP_MUL_MAT_ID:
+            // src0 (weights) is streamed once per output column.
+            return (int64_t) ggml_nbytes(node->src[0]) * std::max<int64_t>(node->ne[1], 1);
+        case GGML_OP_FLASH_ATTN_EXT:
+            // K and V are re-read for every q row.
+            return ((int64_t) ggml_nbytes(node->src[1]) + (int64_t) ggml_nbytes(node->src[2])) *
+                   std::max<int64_t>(node->src[0]->ne[1], 1);
+        default:
+            return (int64_t) ggml_nbytes(node);
+    }
+}
+
 static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {
     ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;
 
+    // QVAC-21914: bound the driver's per-submission work on giant graphs.
+    // Large vision graphs (e.g. a monolithic 16k-patch ViT encode: thousands
+    // of nodes, ~48 s of GPU work) enqueue everything between two host
+    // synchronization points. On Adreno the GSL command-buffer manager
+    // accumulates the whole stream and the GPU faults near the tail
+    // (log_gpu_snapshot), after which the next submission aborts the process
+    // from cl_a8x_cmdbuf_mgr_submit_ibs (os_exit) — observed on Galaxy S25
+    // Ultra / Adreno 830. Flushing whenever the estimated enqueued work
+    // exceeds flush_work_budget hands the driver bounded batches instead.
+    // Gating on WORK (not a node count) scales the flush cadence to the batch
+    // size: the ~48 s encode flushes many times, while a per-token LLM decode
+    // (its work ~= the model size streamed once, i.e. a few flushes per token
+    // at the default budget for a multi-GB model) is far lighter. clFlush only
+    // submits (no host stall), so the decode-path cost is negligible in
+    // practice (measured GPU decode TPS within noise of pre-hardening), but it
+    // is NOT literally zero for large models — raise flush_work_budget past the
+    // model size, or set it to 0, to make decode fully submission-free.
+    int64_t work_since_flush = 0;
+
     for (int i = 0; i < cgraph->n_nodes; i++) {
         ggml_tensor * node = cgraph->nodes[i];
 
@@ -4250,27 +4328,39 @@ static ggml_status ggml_backend_opencl_graph_compute(ggml_backend_t backend, ggm
             continue;
         }
 
+        // Accumulate this node's estimated work before dispatch; the budget
+        // check + flush runs AFTER the node is enqueued (bottom of the loop),
+        // so the node that crosses the budget is part of the flushed batch
+        // rather than deferred into a fresh, potentially-unflushed final
+        // batch. Fused ops are accounted by their anchor node — precision is
+        // irrelevant here (see the QVAC-21914 note above).
+        if (backend_ctx->flush_work_budget > 0) {
+            work_since_flush += ggml_opencl_node_work_estimate(node);
+        }
+
+        // if/else (not `continue`) so every dispatch path reaches the single
+        // budget-flush touch point below.
         if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) {
             ggml_opencl_op_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]);
             i += 2;
-            continue;
-        }
-        if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD })) {
+        } else if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD })) {
             ggml_opencl_op_group_norm_fused(backend, node, cgraph->nodes[i+1], cgraph->nodes[i+2]);
             i += 2;
-            continue;
-        }
-        if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {
+        } else if (!backend_ctx->disable_fusion && ggml_opencl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) {
             ggml_opencl_op_rms_norm_fused(backend, node, cgraph->nodes[i+1]);
             i++;
-            continue;
+        } else {
+            bool ok = ggml_cl_compute_forward(backend, node);
+            if (!ok) {
+                GGML_LOG_ERROR("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op));
+            }
+            GGML_ASSERT(ok);
         }
 
-        bool ok = ggml_cl_compute_forward(backend, node);
-        if (!ok) {
-            GGML_LOG_ERROR("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op));
+        if (backend_ctx->flush_work_budget > 0 && work_since_flush >= backend_ctx->flush_work_budget) {
+            CL_CHECK(clFlush(backend_ctx->queue));
+            work_since_flush = 0;
         }
-        GGML_ASSERT(ok);
     }
 
     return GGML_STATUS_SUCCESS;
@@ -9786,6 +9876,11 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
 
     ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;
 
+    // QVAC-21914: n_q is int; the q-chunk loop below accumulates into an int
+    // (q_base += chunk_rows) and derives cl_ulong byte offsets from it. Guard
+    // the int64->int truncation so a pathological q-row count can't wrap
+    // negative and produce an out-of-bounds device offset.
+    GGML_ASSERT(q->ne[1] <= INT32_MAX);
     const int n_q = q->ne[1];
     const int n_kv = k->ne[1];
     const int d_head_q = q->ne[0];
@@ -9922,17 +10017,61 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
     CL_CHECK(clSetKernelArg(kernel, 38, sizeof(cl_mem),   &sinks_buffer));
     CL_CHECK(clSetKernelArg(kernel, 39, sizeof(cl_ulong), &offset_sinks));
 
+    if (n_q == 0) {
+        // Degenerate empty dispatch: nothing to compute. Guard explicitly so
+        // the chunk loop below cannot be entered with a zero row count.
+        return;
+    }
+
     if (n_q == 1) {
         const size_t wg_size = 64;
         size_t local_work_size[] = { wg_size, 1 };
         size_t global_work_size[] = { wg_size, (size_t)(n_head * n_batch) };
         backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst);
     } else {
+        // QVAC-21914: chunk very large dispatches along the q-rows so no
+        // single kernel runs unboundedly long. A monolithic ViT encode at
+        // image_max_tokens=4096 puts n_q = n_kv = 16384 through one dispatch
+        // whose every workgroup loops the full 16k KV — on Adreno 830
+        // (Galaxy S25 Ultra, OpenCL) the GPU faults near the end of such
+        // encodes and the driver then kills the process on the next
+        // submission (cl_a8x_cmdbuf_mgr_submit_ibs -> os_exit). Splitting is
+        // exact: the kernel resolves its q row as
+        // group_id(0)*BLOCK_M + tid relative to the Q/O/mask base offsets,
+        // is_causal is always 0 (masking is explicit) and alibi/sinks depend
+        // on the head index only, so shifting the row base via the byte
+        // offsets while shrinking n_q is mathematically identical. clFlush
+        // between chunks hands the driver bounded submissions; 0 disables
+        // (GGML_OPENCL_FA_MAX_NQ, resolved at init).
+        //
+        // The kernel's causal-boundary formula needs the TOTAL n_q; chunks
+        // after the first would silently corrupt output if causal FA were
+        // ever enabled here. This backend always passes explicit masks
+        // (is_causal == 0) — keep it loud if that invariant ever changes.
+        GGML_ASSERT(is_causal == 0 && "FA q-chunking requires total n_q for the causal boundary");
+        const int fa_max_nq = backend_ctx->fa_max_nq;
         const int block_m = backend_ctx->kernels_flash_attn_bm.at(dk_dv);
         const size_t wg_size = block_m;
-        size_t local_work_size[] = { wg_size, 1 };
-        size_t global_work_size[] = { (size_t)((n_q + block_m - 1) / block_m) * wg_size, (size_t)(n_head * n_batch) };
-        backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst);
+        const int chunk_rows = (fa_max_nq > 0 && n_q > fa_max_nq) ? fa_max_nq : n_q;
+
+        for (int q_base = 0; q_base < n_q; q_base += chunk_rows) {
+            const int n_q_chunk = std::min(chunk_rows, n_q - q_base);
+            if (q_base > 0 || n_q_chunk != n_q) {
+                const cl_ulong offset_q_chunk    = offset_q + (cl_ulong)q_base * q_nb1;
+                const cl_ulong offset_o_chunk    = offset_o + (cl_ulong)q_base * o_nb1;
+                const cl_ulong offset_mask_chunk = mask ? offset_mask + (cl_ulong)q_base * mask_nb1 : offset_mask;
+                CL_CHECK(clSetKernelArg(kernel, 1,  sizeof(cl_ulong), &offset_q_chunk));
+                CL_CHECK(clSetKernelArg(kernel, 7,  sizeof(cl_ulong), &offset_o_chunk));
+                CL_CHECK(clSetKernelArg(kernel, 9,  sizeof(int),      &n_q_chunk));
+                CL_CHECK(clSetKernelArg(kernel, 32, sizeof(cl_ulong), &offset_mask_chunk));
+            }
+            size_t local_work_size[] = { wg_size, 1 };
+            size_t global_work_size[] = { (size_t)((n_q_chunk + block_m - 1) / block_m) * wg_size, (size_t)(n_head * n_batch) };
+            backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size, local_work_size, dst);
+            if (q_base + chunk_rows < n_q) {
+                CL_CHECK(clFlush(backend_ctx->queue));
+            }
+        }
     }
 }
 

tests/CMakeLists.txt

@@ -261,6 +261,13 @@ llama_build_and_test(test-backend-ops.cpp)
 # subgroup size. Auto-skips when no GPU backend is available.
 llama_build_and_test(test-copy-tbq-subgroups.cpp)
 
+# QVAC-21914: ggml-opencl flash-attention q-chunking parity vs the CPU
+# backend. Unconditional and self-skipping at runtime when no OpenCL device is
+# present (like test-copy-tbq-subgroups above), so it exercises chunking
+# wherever OpenCL exists (Adreno devices, desktop OpenCL, PoCL). Does not link
+# mtmd, so it is intentionally outside the LLAMA_MTMD block below.
+llama_build_and_test(test-opencl-fa-chunking.cpp)
+
 llama_build_and_test(test-model-load-cancel.cpp       LABEL "model")
 llama_build_and_test(test-model-load-disk.cpp         LABEL "model")
 llama_build_and_test(test-model-load-memory.cpp       LABEL "model")
@@ -297,6 +304,13 @@ if (LLAMA_MTMD)
     llama_build_and_test(test-mtmd-c-api.c)
     target_link_libraries(${LLAMA_TEST_NAME} PRIVATE mtmd)
     unset(LLAMA_TEST_NAME)
+
+    # QVAC-21914: clip flash-attention AUTO budget arithmetic (pure CPU).
+    set(LLAMA_TEST_NAME test-clip-fa-cutoff)
+    llama_build_and_test(test-clip-fa-cutoff.cpp)
+    target_link_libraries(${LLAMA_TEST_NAME} PRIVATE mtmd)
+    target_include_directories(${LLAMA_TEST_NAME} PRIVATE ${PROJECT_SOURCE_DIR}/tools/mtmd)
+    unset(LLAMA_TEST_NAME)
 endif()
 
 # GGUF model data fetcher library for tests that need real model metadata

tests/test-clip-fa-cutoff.cpp

@@ -0,0 +1,88 @@
+// QVAC-21914: unit test for the clip flash-attention AUTO budget arithmetic
+// (clip_fa_effective_min_kv in tools/mtmd/clip.cpp — pure function, exposed
+// via clip.h). Locks in both halves of the design:
+//   - the fast explicit path survives momentary memory pressure (the cutoff
+//     is clamped by STABLE total memory, and free memory may only lower it
+//     by the hard-fit requirement — never the old free/2 heuristic), and
+//   - the OOM guard fails SAFE: unknown memory info caps the cutoff at a
+//     conservative constant instead of trusting the raw default.
+
+#include "clip.h"
+
+#include <cstdio>
+#include <cstdlib>
+
+static int g_failures = 0;
+
+static void expect_eq(const char * what, int got, int expected) {
+    if (got != expected) {
+        std::printf("FAIL %s: got %d, expected %d\n", what, got, expected);
+        g_failures++;
+    } else {
+        std::printf("ok   %s: %d\n", what, got);
+    }
+}
+
+static void expect_range(const char * what, int got, int lo, int hi) {
+    if (got < lo || got > hi) {
+        std::printf("FAIL %s: got %d, expected in [%d, %d]\n", what, got, lo, hi);
+        g_failures++;
+    } else {
+        std::printf("ok   %s: %d in [%d, %d]\n", what, got, lo, hi);
+    }
+}
+
+int main() {
+    const size_t GB = 1000ull * 1000ull * 1000ull;
+
+    // Cutoff disabled (or negative) passes through untouched — AUTO stays in
+    // legacy "FA whenever supported" mode regardless of memory info.
+    expect_eq("disabled cutoff, no mem",        clip_fa_effective_min_kv(0,    0,       0,      16), 0);
+    expect_eq("disabled cutoff, with mem",      clip_fa_effective_min_kv(0,    16 * GB, 8 * GB, 16), 0);
+    expect_eq("negative cutoff passes through", clip_fa_effective_min_kv(-1,   16 * GB, 0,      16), -1);
+
+    // Plentiful total memory: the 4096 default survives (16 GB, n_head=16:
+    // total clamp = sqrt((16e9/2)/(24*16)) ~= 4564 > 4096).
+    expect_eq("16GB total keeps default", clip_fa_effective_min_kv(4096, 16 * GB, 0, 16), 4096);
+
+    // 12 GB-class device (Pixel 9-class): total clamp ~= sqrt((12e9/2)/384)
+    // ~= 3952 — trims only the topmost band of the default.
+    expect_range("12GB total trims to ~3950", clip_fa_effective_min_kv(4096, 12 * GB, 0, 16), 3900, 4000);
+
+    // P2 regression guard: high total + momentarily low free must NOT fall
+    // back to the old volatile free/2 heuristic (~1976 at 1.5 GB free). The
+    // hard-fit bound sqrt(1.5e9/(12*16)) ~= 2795 is the correct floor.
+    expect_range("1.5GB free -> hard-fit, not free/2",
+                 clip_fa_effective_min_kv(4096, 16 * GB, (size_t)(1.5 * (double) GB), 16), 2700, 2900);
+
+    // Ample free memory does not restrict below the total clamp...
+    expect_eq("8GB free does not restrict", clip_fa_effective_min_kv(4096, 16 * GB, 8 * GB, 16), 4096);
+    // ...and free memory can never RAISE the cutoff past the total clamp.
+    expect_range("huge free cannot raise past total clamp",
+                 clip_fa_effective_min_kv(4096, 2 * GB, 64 * GB, 16), 1550, 1700);
+
+    // No memory info at all: fail SAFE at the conservative constant cap
+    // (2048), never the raw 4096 default (~3.2 GB scratch at n_head=16).
+    expect_eq("no meminfo caps at 2048",        clip_fa_effective_min_kv(4096, 0, 0, 16), 2048);
+    expect_eq("no meminfo keeps smaller cutoff", clip_fa_effective_min_kv(1000, 0, 0, 16), 1000);
+
+    // Degenerate n_head is guarded (treated as 1), not a crash/div-by-zero.
+    // Must pass memory info so a sqrt(.../n_head) branch actually runs — with
+    // total_mem==free_mem==0 the NO_MEMINFO short-circuit returns before the
+    // guard is reached, leaving it untested. At 16 GB total, n_head=1:
+    // total clamp = sqrt((16e9/2)/24) ~= 18257 > 4096, so the default 4096
+    // survives (and the run completing at all proves no div-by-zero).
+    expect_eq("n_head=0 guarded (total mem path)", clip_fa_effective_min_kv(4096, 16 * GB, 0, 0), 4096);
+
+    // Fewer heads => larger permissible n (scratch is linear in n_head):
+    // same 1.5 GB free, n_head=4 -> sqrt(1.5e9/48) ~= 5590, so the 4096
+    // default survives untouched.
+    expect_eq("fewer heads relax the fit bound", clip_fa_effective_min_kv(4096, 16 * GB, (size_t)(1.5 * (double) GB), 4), 4096);
+
+    if (g_failures) {
+        std::printf("%d failure(s)\n", g_failures);
+        return 1;
+    }
+    std::printf("all ok\n");
+    return 0;
+}