CUDA: route single-column f32 mul_mat through mmvf via transpose-free operand swap

A mul_mat of a single-column f32 weight (ne01 == 1) against a wide activation
(ne11 > MMVF_MAX_BATCH_SIZE) misses the mmvf vector kernel because ne11 is the
batch dimension, and falls back to an f32 cuBLAS/rocBLAS GEMM whose 32x32 macro
tile computes just one useful output column. Since the output is a vector, the
operands can be swapped transpose-free: the activation becomes the matrix and the
single-column weight the lone F32 vector, with dst reinterpreted as [ne11, 1].

This runs the op as a proper GEMV. The graph is unchanged, so LoRA and any other
consumer are unaffected. On Qwen3.6-35B-A3B (gfx1151, pp128) the shared-expert
gate (weight f32[2048,1]) drops from ~78 us/instance on rocBLAS Sgemm to ~3.8 us
on mmvf (~20x), ~3.0 ms saved across the 40 layers.

Adds test-backend-ops regression cases (m=1, n sweeping across MMVF_MAX_BATCH_SIZE)
that validate both the direct and swapped paths against the CPU reference.

Co-Authored-By: Claude Opus 4 (1M context) <noreply@anthropic.com>

Author: roberteg16

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -1759,6 +1759,27 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
         ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst);
         return;
     }
+    // A single-column f32 weight (ne01 == 1) times a wide activation (ne11 > MMVF_MAX_BATCH_SIZE)
+    // misses mmvf above because ne11 is the batch dim, so it falls back to a wasteful f32 cuBLAS
+    // GEMM. The output is a vector, so swapping the operands is transpose-free: the activation
+    // becomes the matrix and the single-column weight the lone F32 vector, with dst reinterpreted
+    // as [ne11, 1].
+    if (ne01 == 1 && ne11 > MMVF_MAX_BATCH_SIZE && ne2 == 1 && ne3 == 1
+            && src0->type == GGML_TYPE_F32
+            && ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)
+            && ggml_cuda_should_use_mmvf(src1->type, cc, src1->ne, src1->nb, /*ne11 =*/ 1)) {
+        // dst is [ne00=1, ne11] (one output feature, ne11 tokens); the swapped GEMV instead
+        // produces [ne11, 1] (ne11 rows, a single vector column). Both are the same ne11
+        // contiguous floats, so we only relabel the shape of a shallow copy - no data moves:
+        ggml_tensor dst_vec = *dst;
+        dst_vec.ne[0] = ne11;                    // rows: one result per token
+        dst_vec.ne[1] = 1;                       // a single output vector column
+        dst_vec.nb[1] = dst_vec.nb[0]*ne11;      // nb[0] (element stride) stays; row dim spans ne11
+        dst_vec.nb[2] = dst_vec.nb[1];           // higher dims are 1 -> same stride
+        dst_vec.nb[3] = dst_vec.nb[1];
+        ggml_cuda_mul_mat_vec_f(ctx, src1, src0, nullptr, &dst_vec);
+        return;
+    }
     if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11, /*mul_mat_id =*/ false)) {
         ggml_cuda_mul_mat_f(ctx, src0, src1, nullptr, dst);
         return;

tests/test-backend-ops.cpp

@@ -8555,6 +8555,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));
     test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));
 
+    // single-row f32 weight (m == 1) against a wide activation (n): the second column count sweeps
+    // across MMVF_MAX_BATCH_SIZE (8) so both the direct mmvf path (n <= 8) and the CUDA/HIP
+    // operand-swap GEMV path (n > 8, which would otherwise fall back to an f32 cuBLAS GEMM) are
+    // exercised and checked against the CPU reference. Regression guard for the mul_mat_vec_f swap.
+    for (int64_t n : {1, 7, 8, 9, 16, 128, 512}) {
+        test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1, n, 2048, {1, 1}, {1, 1}));
+    }
+
 
 #if 0
     {