QVAC-21320 feat: Mali GPU projector optimizations — disable FA, warptile, layernorm fusion

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -811,6 +811,7 @@ struct vk_device_struct {
     vk_pipeline pipeline_set_rows_i32[GGML_TYPE_COUNT];
     vk_pipeline pipeline_set_rows_i64[GGML_TYPE_COUNT];
     vk_pipeline pipeline_norm_f32;
+    vk_pipeline pipeline_norm_mul_add_f32;
     vk_pipeline pipeline_group_norm_f32;
     vk_pipeline pipeline_rms_norm_f32;
     vk_pipeline pipeline_rms_norm_mul_f32;
@@ -3653,6 +3654,14 @@ static void ggml_vk_load_shaders(vk_device& device) {
             // Xe2/Xe3 with coopmat enabled - warptile performance tuning
             l_warptile = { 512, 128, 128, 16, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 };
             l_warptile_mmq = { 512, 128, 128, 32, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 };
+        } else if (device->vendor_id == VK_VENDOR_ID_ARM) {
+            // QVAC-21257 iter2: Mali/Valhall (16-wide subgroup, no coopmat). The q8_0 MMQ matmuls
+            // dominate the CLIP vision-encode (~65 %, ~90 GFLOPS/s, run #80). The generic large MMQ
+            // tile uses only 8 warps/workgroup (block_size 128); widen to a 32-warp shape (block_size
+            // 512, wm=16/wn=32 — the valid 16-wide layout the Intel Xe2 branch uses) to raise GPU
+            // occupancy on the dominant matmul path. Falls back automatically (shmem check below) if
+            // it doesn't fit. Float MMQ path only (q8_0 goes through mul_mat_q_f16).
+            l_warptile_mmq = { 512, 128, 128, 32, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 };
         }
 
         l_wg_denoms     = { l_warptile[1],     l_warptile[2],     1 };
@@ -5037,7 +5046,11 @@ static void ggml_vk_load_shaders(vk_device& device) {
     }
     ggml_vk_create_pipeline(device, device->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_nc_push_constants), {1, 1, 1}, {}, 1);
 
-    ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1);
+    // norm.comp now uses the binary head (4 bindings: input, weight, bias, output) and
+    // spec constants {norepeat, do_multiply, do_add}. Plain norm sets both to 0; the fused
+    // NORM+MUL+ADD path sets both to 1. Same bytecode for both, mirroring rms_norm.
+    ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0}, 1, true);
+    ggml_vk_create_pipeline(device, device->pipeline_norm_mul_add_f32, "norm_mul_add_f32", norm_f32_len, norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true);
     ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1);
 
     ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true);
@@ -10937,7 +10950,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
         return nullptr;
     case GGML_OP_NORM:
         if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
-            return ctx->device->pipeline_norm_f32;
+            // num_additional_fused_ops == 2 means fused NORM+MUL+ADD (layernorm scale+bias).
+            return ctx->num_additional_fused_ops == 2 ? ctx->device->pipeline_norm_mul_add_f32 : ctx->device->pipeline_norm_f32;
         }
         return nullptr;
     case GGML_OP_GROUP_NORM:
@@ -11504,6 +11518,10 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
 
     switch (op) {
     case GGML_OP_NORM:
+        // Flattened/tiled row dispatch (group below) — norm.comp reconstructs
+        // {row, channel, sample} from the flat workgroup id, so large row counts
+        // never exceed maxComputeWorkGroupCount (a direct {ne01, ne02, ne03}
+        // grid would trip the dispatch assert).
     case GGML_OP_RMS_NORM_BACK:
     case GGML_OP_L2_NORM:
     case GGML_OP_SOFT_MAX:
@@ -12563,10 +12581,51 @@ static void ggml_vk_geglu_back(ggml_backend_vk_context * ctx, vk_context& subctx
     ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_GEGLU_BACK, { (uint32_t)ggml_nelements(dst), (uint32_t)dst->ne[0], 0.0f, 0.0f, 0.0f, 0.0f });
 }
 
-static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
-    float * op_params = (float *)dst->op_params;
+static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
+    ggml_tensor * norm = cgraph->nodes[node_idx];
+    float * op_params = (float *)norm->op_params;
+
+    ggml_tensor * dst;
+    const ggml_tensor * src0; // norm input (A)
+    const ggml_tensor * src1; // weight   (B)
+    const ggml_tensor * src2; // bias     (C)
 
-    ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f, 0.0f, 0.0f });
+    if (ctx->num_additional_fused_ops == 2) {
+        // fused NORM + MUL + ADD (layernorm scale + bias)
+        ggml_tensor * mul = cgraph->nodes[node_idx + 1];
+        ggml_tensor * add = cgraph->nodes[node_idx + 2];
+        ggml_tensor * weight = mul->src[0] == norm ? mul->src[1] : mul->src[0];
+        ggml_tensor * bias   = add->src[0] == mul  ? add->src[1] : add->src[0];
+        // shader uses plain col indexing (no stride), requires zero misalignment
+        GGML_ASSERT(get_misalign_bytes(ctx, weight) == 0);
+        GGML_ASSERT(get_misalign_bytes(ctx, bias)   == 0);
+        dst  = add;
+        src0 = norm->src[0];
+        src1 = weight;
+        src2 = bias;
+    } else {
+        dst  = norm;
+        // plain norm: bind weight/bias to the input so all 4 descriptors are valid.
+        // do_multiply/do_add spec constants are false, so they are never read.
+        src0 = norm->src[0];
+        src1 = norm->src[0];
+        src2 = norm->src[0];
+    }
+
+    const uint32_t src0_type_size = ggml_type_size(src0->type);
+    const uint32_t src1_type_size = ggml_type_size(src1->type);
+    const uint32_t src2_type_size = ggml_type_size(src2->type);
+
+    vk_op_binary_push_constants bin {
+        (uint32_t)ggml_nelements(src0),
+        (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size,
+        (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2], (uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size,
+        (uint32_t)src2->ne[0], (uint32_t)src2->ne[1], (uint32_t)src2->ne[2], (uint32_t)src2->ne[3], (uint32_t)src2->nb[0] / src2_type_size, (uint32_t)src2->nb[1] / src2_type_size, (uint32_t)src2->nb[2] / src2_type_size, (uint32_t)src2->nb[3] / src2_type_size,
+        0,
+        op_params[0], 0.0f, 0,
+    };
+
+    ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_NORM, std::move(bin));
 }
 
 static void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
@@ -14872,7 +14931,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr
 
         break;
     case GGML_OP_NORM:
-        ggml_vk_norm(ctx, compute_ctx, src0, node);
+        ggml_vk_norm(ctx, compute_ctx, cgraph, node_idx);
 
         break;
     case GGML_OP_GROUP_NORM:
@@ -15781,6 +15840,50 @@ static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct g
             return false;
         }
     }
+    if (ops.size() == 3 && ops.begin()[0] == GGML_OP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) {
+        // fused layernorm (NORM) + MUL (scale) + ADD (bias)
+        const ggml_tensor *norm = cgraph->nodes[node_idx];
+        const ggml_tensor *mul  = cgraph->nodes[node_idx + 1];
+        const ggml_tensor *add  = cgraph->nodes[node_idx + 2];
+
+        // f32-only
+        if (norm->src[0]->type != GGML_TYPE_F32 || norm->type != GGML_TYPE_F32 ||
+            mul->type  != GGML_TYPE_F32 ||
+            add->type  != GGML_TYPE_F32) {
+            return false;
+        }
+        // MUL must consume the NORM result as src[0], ADD must consume the MUL result as src[0].
+        if (mul->src[0] != norm || add->src[0] != mul) {
+            return false;
+        }
+        const ggml_tensor *weight = mul->src[1];
+        const ggml_tensor *bias   = add->src[1];
+        if (weight->type != GGML_TYPE_F32 || bias->type != GGML_TYPE_F32) {
+            return false;
+        }
+        // weight/bias must be 1-D row vectors broadcast over rows, matching norm->ne[0].
+        if (weight->ne[0] != norm->ne[0] || bias->ne[0] != norm->ne[0]) {
+            return false;
+        }
+        if (ggml_nrows(weight) != 1 || ggml_nrows(bias) != 1) {
+            return false;
+        }
+        // mul/add outputs must match norm input shape (no broadcast batch dims)
+        if (!ggml_are_same_shape(norm->src[0], add)) {
+            return false;
+        }
+        // contiguous and aligned (shader assumes contiguous rows and zero misalignment)
+        if (!ggml_is_contiguous(weight) || !ggml_is_contiguous(bias)) {
+            return false;
+        }
+        if (!ggml_is_contiguous_rows(norm->src[0])) {
+            return false;
+        }
+        if (get_misalign_bytes(ctx, weight) != 0 || get_misalign_bytes(ctx, bias) != 0) {
+            return false;
+        }
+    }
+
     auto const &mm_add_ok = [&](const ggml_tensor *mul, const ggml_tensor *add) {
         const ggml_tensor *bias = add->src[0] == mul ? add->src[1] : add->src[0];
 
@@ -16337,6 +16440,14 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
                 // they are overwritten, and one workgroup per row. So close enough.
                 op_srcs_fused_elementwise[0] = true;
                 op_srcs_fused_elementwise[1] = true;
+            } else if (ggml_vk_can_fuse(ctx, cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) {
+                ctx->num_additional_fused_ops = 2;
+                fusion_string = "NORM_MUL_ADD";
+                // norm is not elementwise, but whole rows are consumed by one workgroup per
+                // row and the mean/variance are computed before output is written. So close enough.
+                op_srcs_fused_elementwise[0] = true;
+                op_srcs_fused_elementwise[1] = true;
+                op_srcs_fused_elementwise[2] = true;
             } else if (ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }, { i + 2 }) &&
                        ggml_check_edges(cgraph, i, rope_view_set_rows_edges) &&
                        ggml_vk_can_fuse_rope_set_rows(ctx, cgraph, i)) {
@@ -16665,6 +16776,10 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
                 if (!used[c] &&
                     is_src_of(graph->nodes[j], graph->nodes[c]) &&
                     !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_RMS_NORM && graph->nodes[j]->op == GGML_OP_MUL) &&
+                    !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_NORM && graph->nodes[j]->op == GGML_OP_MUL) &&
+                    // Keep NORM->MUL->ADD consecutive: allow MUL->ADD only when MUL follows a NORM.
+                    !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL && graph->nodes[j]->op == GGML_OP_ADD &&
+                      current_set.size() >= 2 && graph->nodes[current_set[current_set.size() - 2]]->op == GGML_OP_NORM) &&
                     !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT && graph->nodes[j]->op == GGML_OP_ADD) &&
                     !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_ADD_ID) &&
                     !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_MUL) &&
@@ -17949,11 +18064,31 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg,
     return devices[device];
 }
 
+static bool ggml_backend_vk_supports_efficient_fa(ggml_backend_t backend) {
+    ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
+    // ARM/Mali (Valhall) advertises VK_KHR_cooperative_matrix (so coopmat1_fa_support
+    // is true), but its flash-attention still runs the slow path (~40 GFLOPS/s vs the
+    // ~100 GFLOPS/s matmul path; QVAC-21257 profiling). Coopmat-present != efficient FA
+    // here — treat Mali as having no efficient FA so the CLIP projector disables it.
+    if (ctx->device->vendor_id == VK_VENDOR_ID_ARM) {
+        return false;
+    }
+    return ctx->device->coopmat2 || ctx->device->coopmat1_fa_support;
+}
+
+static void * ggml_backend_vk_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {
+    GGML_UNUSED(reg);
+    if (strcmp(name, "ggml_backend_supports_efficient_fa") == 0) {
+        return (void *)ggml_backend_vk_supports_efficient_fa;
+    }
+    return NULL;
+}
+
 static const struct ggml_backend_reg_i ggml_backend_vk_reg_i = {
     /* .get_name         = */ ggml_backend_vk_reg_get_name,
     /* .get_device_count = */ ggml_backend_vk_reg_get_device_count,
     /* .get_device       = */ ggml_backend_vk_reg_get_device,
-    /* .get_proc_address = */ NULL,
+    /* .get_proc_address = */ ggml_backend_vk_reg_get_proc_address,
 };
 
 ggml_backend_reg_t ggml_backend_vk_reg() {

ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl

@@ -19,7 +19,7 @@ layout (push_constant) uniform parameter
 #endif
 } p;
 
-#if !RMS_NORM_ROPE_FUSION
+#if !RMS_NORM_ROPE_FUSION && !NORM_MUL_ADD_FUSION
 layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
 #if defined(A_TYPE_PACKED16)
 layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16[];};

ggml/src/ggml-vulkan/vulkan-shaders/norm.comp

@@ -1,44 +1,95 @@
 #version 450
 
-#include "generic_head.glsl"
+#define NORM_MUL_ADD_FUSION 1
+
+#include "generic_binary_head.glsl"
 #include "types.glsl"
 
 #extension GL_EXT_control_flow_attributes : enable
 #define BLOCK_SIZE 512
 
+// Spec constant 0 (norepeat) is declared in generic_binary_head.glsl.
+layout (constant_id = 1) const bool do_multiply = false;
+layout (constant_id = 2) const bool do_add      = false;
+
 layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
 
-layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
-layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
+// Bindings: 0=input(A), 1=weight(B), 2=bias(C), 3=output(D).
+// When NORM_MUL_ADD_FUSION is set, generic_binary_head.glsl does not declare
+// any bindings, so we declare all four here. The weight/bias buffers are only
+// read when do_multiply/do_add are true; otherwise they may alias the input.
+layout (binding = 0) readonly  buffer A {A_TYPE data_a[];};
+layout (binding = 1) readonly  buffer B {B_TYPE data_b[];};
+layout (binding = 2) readonly  buffer C {B_TYPE data_c[];};
+layout (binding = 3) writeonly buffer D {D_TYPE data_d[];};
 
-shared vec2 sum[BLOCK_SIZE];
+shared vec2 sumsh[BLOCK_SIZE];
 
 void main() {
-    const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x;
-    const uint tid = gl_LocalInvocationID.x;
+    const uint ncols     = p.ne00;
+    const uint nrows     = p.ne01;
+    const uint nchannels = p.ne02;
+
+    // The host dispatches the flattened/tiled {512, 512, N} row grid (same as
+    // soft_max.comp et al.) so huge row counts never exceed
+    // maxComputeWorkGroupCount; reconstruct {row, channel, sample} from the
+    // flat workgroup id.
+    const uint flat_row  = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x;
+    if (flat_row >= nrows * nchannels * p.ne03) {
+        // Tiling round-up padding; uniform across the workgroup, so returning
+        // before the barriers below is safe.
+        return;
+    }
+    const uint row       = flat_row % nrows;
+    const uint channel   = (flat_row / nrows) % nchannels;
+    const uint samp      = flat_row / (nrows * nchannels);
+    const uint tid       = gl_LocalInvocationID.x;
 
-    sum[tid] = vec2(0.0f, 0.0f);
+    const uint stride_row     = p.nb01;
+    const uint stride_channel = p.nb02;
+    const uint stride_sample  = p.nb03;
 
-    [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) {
-        const float xi = float(data_a[row*p.KX + col]);
-        sum[tid].x += xi;
-        sum[tid].y += xi * xi;
+    // Input offset mirrors rms_norm.comp's stride-based scheme.
+    uint32_t a_offset = samp*stride_sample + channel*stride_channel + row*stride_row + get_aoffset();
+    uint32_t b_offset = get_boffset();
+    // Bias (C) is required to be aligned (misalign==0) by the fusion gate.
+    uint32_t c_offset = 0;
+    // Output is contiguous per row (CLIP layernorm dst); flat_row equals
+    // (samp*nchannels + channel)*nrows + row by construction.
+    uint32_t d_offset = flat_row*ncols + get_doffset();
+
+    vec2 sum = vec2(0.0f, 0.0f);
+
+    [[unroll]] for (uint col = tid; col < ncols; col += BLOCK_SIZE) {
+        const float xi = float(data_a[a_offset + col]);
+        sum.x += xi;
+        sum.y += xi * xi;
     }
 
+    sumsh[tid] = sum;
     // sum up partial sums and write back result
     barrier();
     [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
         if (tid < s) {
-            sum[tid] += sum[tid + s];
+            sum += sumsh[tid + s];
+            sumsh[tid] = sum;
         }
         barrier();
     }
+    sum = sumsh[0];
 
-    const float mean = sum[0].x / p.KX;
-    const float var = sum[0].y / p.KX - mean * mean;
+    const float mean    = sum.x / float(ncols);
+    const float var     = sum.y / float(ncols) - mean * mean;
     const float inv_std = inversesqrt(var + p.param1);
 
-    [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) {
-        data_d[row*p.KX + col] = D_TYPE((float(data_a[row*p.KX + col]) - mean) * inv_std);
+    [[unroll]] for (uint col = tid; col < ncols; col += BLOCK_SIZE) {
+        float result = (float(data_a[a_offset + col]) - mean) * inv_std;
+        if (do_multiply) {
+            result *= float(data_b[b_offset + col]);
+        }
+        if (do_add) {
+            result += float(data_c[c_offset + col]);
+        }
+        data_d[d_offset + col] = D_TYPE(result);
     }
 }

ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp

@@ -994,7 +994,7 @@ void process_shaders() {
     string_to_spv("mul_mat_vec_nc_f16_f32", "mul_mat_vec_nc.comp", {{"A_TYPE", "float16_t"}, {"A_TYPEV4", "f16vec4"}, {"B_TYPE", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}});
 
     // Norms
-    string_to_spv("norm_f32", "norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
+    string_to_spv("norm_f32", "norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}));
     string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
     string_to_spv("rms_norm_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}));
     string_to_spv("rms_norm_partials_f32", "rms_norm_partials.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}}));