diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 5f76ab661a4f..84c9d525e269 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -811,6 +811,7 @@ struct vk_device_struct { vk_pipeline pipeline_set_rows_i32[GGML_TYPE_COUNT]; vk_pipeline pipeline_set_rows_i64[GGML_TYPE_COUNT]; vk_pipeline pipeline_norm_f32; + vk_pipeline pipeline_norm_mul_add_f32; vk_pipeline pipeline_group_norm_f32; vk_pipeline pipeline_rms_norm_f32; vk_pipeline pipeline_rms_norm_mul_f32; @@ -3653,6 +3654,14 @@ static void ggml_vk_load_shaders(vk_device& device) { // Xe2/Xe3 with coopmat enabled - warptile performance tuning l_warptile = { 512, 128, 128, 16, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 }; l_warptile_mmq = { 512, 128, 128, 32, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 }; + } else if (device->vendor_id == VK_VENDOR_ID_ARM) { + // QVAC-21257 iter2: Mali/Valhall (16-wide subgroup, no coopmat). The q8_0 MMQ matmuls + // dominate the CLIP vision-encode (~65 %, ~90 GFLOPS/s, run #80). The generic large MMQ + // tile uses only 8 warps/workgroup (block_size 128); widen to a 32-warp shape (block_size + // 512, wm=16/wn=32 — the valid 16-wide layout the Intel Xe2 branch uses) to raise GPU + // occupancy on the dominant matmul path. Falls back automatically (shmem check below) if + // it doesn't fit. Float MMQ path only (q8_0 goes through mul_mat_q_f16). + l_warptile_mmq = { 512, 128, 128, 32, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 }; } l_wg_denoms = { l_warptile[1], l_warptile[2], 1 }; @@ -5037,7 +5046,11 @@ static void ggml_vk_load_shaders(vk_device& device) { } ggml_vk_create_pipeline(device, device->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_nc_push_constants), {1, 1, 1}, {}, 1); - ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); + // norm.comp now uses the binary head (4 bindings: input, weight, bias, output) and + // spec constants {norepeat, do_multiply, do_add}. Plain norm sets both to 0; the fused + // NORM+MUL+ADD path sets both to 1. Same bytecode for both, mirroring rms_norm. + ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0}, 1, true); + ggml_vk_create_pipeline(device, device->pipeline_norm_mul_add_f32, "norm_mul_add_f32", norm_f32_len, norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true); @@ -10937,7 +10950,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const return nullptr; case GGML_OP_NORM: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { - return ctx->device->pipeline_norm_f32; + // num_additional_fused_ops == 2 means fused NORM+MUL+ADD (layernorm scale+bias). + return ctx->num_additional_fused_ops == 2 ? ctx->device->pipeline_norm_mul_add_f32 : ctx->device->pipeline_norm_f32; } return nullptr; case GGML_OP_GROUP_NORM: @@ -11504,6 +11518,10 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co switch (op) { case GGML_OP_NORM: + // Flattened/tiled row dispatch (group below) — norm.comp reconstructs + // {row, channel, sample} from the flat workgroup id, so large row counts + // never exceed maxComputeWorkGroupCount (a direct {ne01, ne02, ne03} + // grid would trip the dispatch assert). case GGML_OP_RMS_NORM_BACK: case GGML_OP_L2_NORM: case GGML_OP_SOFT_MAX: @@ -12563,10 +12581,51 @@ static void ggml_vk_geglu_back(ggml_backend_vk_context * ctx, vk_context& subctx ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_GEGLU_BACK, { (uint32_t)ggml_nelements(dst), (uint32_t)dst->ne[0], 0.0f, 0.0f, 0.0f, 0.0f }); } -static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { - float * op_params = (float *)dst->op_params; +static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) { + ggml_tensor * norm = cgraph->nodes[node_idx]; + float * op_params = (float *)norm->op_params; + + ggml_tensor * dst; + const ggml_tensor * src0; // norm input (A) + const ggml_tensor * src1; // weight (B) + const ggml_tensor * src2; // bias (C) - ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f, 0.0f, 0.0f }); + if (ctx->num_additional_fused_ops == 2) { + // fused NORM + MUL + ADD (layernorm scale + bias) + ggml_tensor * mul = cgraph->nodes[node_idx + 1]; + ggml_tensor * add = cgraph->nodes[node_idx + 2]; + ggml_tensor * weight = mul->src[0] == norm ? mul->src[1] : mul->src[0]; + ggml_tensor * bias = add->src[0] == mul ? add->src[1] : add->src[0]; + // shader uses plain col indexing (no stride), requires zero misalignment + GGML_ASSERT(get_misalign_bytes(ctx, weight) == 0); + GGML_ASSERT(get_misalign_bytes(ctx, bias) == 0); + dst = add; + src0 = norm->src[0]; + src1 = weight; + src2 = bias; + } else { + dst = norm; + // plain norm: bind weight/bias to the input so all 4 descriptors are valid. + // do_multiply/do_add spec constants are false, so they are never read. + src0 = norm->src[0]; + src1 = norm->src[0]; + src2 = norm->src[0]; + } + + const uint32_t src0_type_size = ggml_type_size(src0->type); + const uint32_t src1_type_size = ggml_type_size(src1->type); + const uint32_t src2_type_size = ggml_type_size(src2->type); + + vk_op_binary_push_constants bin { + (uint32_t)ggml_nelements(src0), + (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2], (uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, + (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2], (uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, + (uint32_t)src2->ne[0], (uint32_t)src2->ne[1], (uint32_t)src2->ne[2], (uint32_t)src2->ne[3], (uint32_t)src2->nb[0] / src2_type_size, (uint32_t)src2->nb[1] / src2_type_size, (uint32_t)src2->nb[2] / src2_type_size, (uint32_t)src2->nb[3] / src2_type_size, + 0, + op_params[0], 0.0f, 0, + }; + + ggml_vk_op_f32(ctx, subctx, src0, src1, src2, nullptr, dst, GGML_OP_NORM, std::move(bin)); } static void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { @@ -14872,7 +14931,7 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr break; case GGML_OP_NORM: - ggml_vk_norm(ctx, compute_ctx, src0, node); + ggml_vk_norm(ctx, compute_ctx, cgraph, node_idx); break; case GGML_OP_GROUP_NORM: @@ -15781,6 +15840,50 @@ static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct g return false; } } + if (ops.size() == 3 && ops.begin()[0] == GGML_OP_NORM && ops.begin()[1] == GGML_OP_MUL && ops.begin()[2] == GGML_OP_ADD) { + // fused layernorm (NORM) + MUL (scale) + ADD (bias) + const ggml_tensor *norm = cgraph->nodes[node_idx]; + const ggml_tensor *mul = cgraph->nodes[node_idx + 1]; + const ggml_tensor *add = cgraph->nodes[node_idx + 2]; + + // f32-only + if (norm->src[0]->type != GGML_TYPE_F32 || norm->type != GGML_TYPE_F32 || + mul->type != GGML_TYPE_F32 || + add->type != GGML_TYPE_F32) { + return false; + } + // MUL must consume the NORM result as src[0], ADD must consume the MUL result as src[0]. + if (mul->src[0] != norm || add->src[0] != mul) { + return false; + } + const ggml_tensor *weight = mul->src[1]; + const ggml_tensor *bias = add->src[1]; + if (weight->type != GGML_TYPE_F32 || bias->type != GGML_TYPE_F32) { + return false; + } + // weight/bias must be 1-D row vectors broadcast over rows, matching norm->ne[0]. + if (weight->ne[0] != norm->ne[0] || bias->ne[0] != norm->ne[0]) { + return false; + } + if (ggml_nrows(weight) != 1 || ggml_nrows(bias) != 1) { + return false; + } + // mul/add outputs must match norm input shape (no broadcast batch dims) + if (!ggml_are_same_shape(norm->src[0], add)) { + return false; + } + // contiguous and aligned (shader assumes contiguous rows and zero misalignment) + if (!ggml_is_contiguous(weight) || !ggml_is_contiguous(bias)) { + return false; + } + if (!ggml_is_contiguous_rows(norm->src[0])) { + return false; + } + if (get_misalign_bytes(ctx, weight) != 0 || get_misalign_bytes(ctx, bias) != 0) { + return false; + } + } + auto const &mm_add_ok = [&](const ggml_tensor *mul, const ggml_tensor *add) { const ggml_tensor *bias = add->src[0] == mul ? add->src[1] : add->src[0]; @@ -16337,6 +16440,14 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // they are overwritten, and one workgroup per row. So close enough. op_srcs_fused_elementwise[0] = true; op_srcs_fused_elementwise[1] = true; + } else if (ggml_vk_can_fuse(ctx, cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD })) { + ctx->num_additional_fused_ops = 2; + fusion_string = "NORM_MUL_ADD"; + // norm is not elementwise, but whole rows are consumed by one workgroup per + // row and the mean/variance are computed before output is written. So close enough. + op_srcs_fused_elementwise[0] = true; + op_srcs_fused_elementwise[1] = true; + op_srcs_fused_elementwise[2] = true; } else if (ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_ROPE, GGML_OP_VIEW, GGML_OP_SET_ROWS }, { i + 2 }) && ggml_check_edges(cgraph, i, rope_view_set_rows_edges) && ggml_vk_can_fuse_rope_set_rows(ctx, cgraph, i)) { @@ -16665,6 +16776,10 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * if (!used[c] && is_src_of(graph->nodes[j], graph->nodes[c]) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_RMS_NORM && graph->nodes[j]->op == GGML_OP_MUL) && + !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_NORM && graph->nodes[j]->op == GGML_OP_MUL) && + // Keep NORM->MUL->ADD consecutive: allow MUL->ADD only when MUL follows a NORM. + !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL && graph->nodes[j]->op == GGML_OP_ADD && + current_set.size() >= 2 && graph->nodes[current_set[current_set.size() - 2]]->op == GGML_OP_NORM) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT && graph->nodes[j]->op == GGML_OP_ADD) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_ADD_ID) && !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_MUL) && @@ -17949,11 +18064,31 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg, return devices[device]; } +static bool ggml_backend_vk_supports_efficient_fa(ggml_backend_t backend) { + ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context; + // ARM/Mali (Valhall) advertises VK_KHR_cooperative_matrix (so coopmat1_fa_support + // is true), but its flash-attention still runs the slow path (~40 GFLOPS/s vs the + // ~100 GFLOPS/s matmul path; QVAC-21257 profiling). Coopmat-present != efficient FA + // here — treat Mali as having no efficient FA so the CLIP projector disables it. + if (ctx->device->vendor_id == VK_VENDOR_ID_ARM) { + return false; + } + return ctx->device->coopmat2 || ctx->device->coopmat1_fa_support; +} + +static void * ggml_backend_vk_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { + GGML_UNUSED(reg); + if (strcmp(name, "ggml_backend_supports_efficient_fa") == 0) { + return (void *)ggml_backend_vk_supports_efficient_fa; + } + return NULL; +} + static const struct ggml_backend_reg_i ggml_backend_vk_reg_i = { /* .get_name = */ ggml_backend_vk_reg_get_name, /* .get_device_count = */ ggml_backend_vk_reg_get_device_count, /* .get_device = */ ggml_backend_vk_reg_get_device, - /* .get_proc_address = */ NULL, + /* .get_proc_address = */ ggml_backend_vk_reg_get_proc_address, }; ggml_backend_reg_t ggml_backend_vk_reg() { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl index dc657f3c7084..ada01ad226fa 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/generic_binary_head.glsl @@ -19,7 +19,7 @@ layout (push_constant) uniform parameter #endif } p; -#if !RMS_NORM_ROPE_FUSION +#if !RMS_NORM_ROPE_FUSION && !NORM_MUL_ADD_FUSION layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; #if defined(A_TYPE_PACKED16) layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16[];}; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp index cc3ea0b76060..5cdc67d51dbe 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp @@ -1,44 +1,95 @@ #version 450 -#include "generic_head.glsl" +#define NORM_MUL_ADD_FUSION 1 + +#include "generic_binary_head.glsl" #include "types.glsl" #extension GL_EXT_control_flow_attributes : enable #define BLOCK_SIZE 512 +// Spec constant 0 (norepeat) is declared in generic_binary_head.glsl. +layout (constant_id = 1) const bool do_multiply = false; +layout (constant_id = 2) const bool do_add = false; + layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; -layout (binding = 0) readonly buffer X {A_TYPE data_a[];}; -layout (binding = 1) writeonly buffer D {D_TYPE data_d[];}; +// Bindings: 0=input(A), 1=weight(B), 2=bias(C), 3=output(D). +// When NORM_MUL_ADD_FUSION is set, generic_binary_head.glsl does not declare +// any bindings, so we declare all four here. The weight/bias buffers are only +// read when do_multiply/do_add are true; otherwise they may alias the input. +layout (binding = 0) readonly buffer A {A_TYPE data_a[];}; +layout (binding = 1) readonly buffer B {B_TYPE data_b[];}; +layout (binding = 2) readonly buffer C {B_TYPE data_c[];}; +layout (binding = 3) writeonly buffer D {D_TYPE data_d[];}; -shared vec2 sum[BLOCK_SIZE]; +shared vec2 sumsh[BLOCK_SIZE]; void main() { - const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; - const uint tid = gl_LocalInvocationID.x; + const uint ncols = p.ne00; + const uint nrows = p.ne01; + const uint nchannels = p.ne02; + + // The host dispatches the flattened/tiled {512, 512, N} row grid (same as + // soft_max.comp et al.) so huge row counts never exceed + // maxComputeWorkGroupCount; reconstruct {row, channel, sample} from the + // flat workgroup id. + const uint flat_row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x; + if (flat_row >= nrows * nchannels * p.ne03) { + // Tiling round-up padding; uniform across the workgroup, so returning + // before the barriers below is safe. + return; + } + const uint row = flat_row % nrows; + const uint channel = (flat_row / nrows) % nchannels; + const uint samp = flat_row / (nrows * nchannels); + const uint tid = gl_LocalInvocationID.x; - sum[tid] = vec2(0.0f, 0.0f); + const uint stride_row = p.nb01; + const uint stride_channel = p.nb02; + const uint stride_sample = p.nb03; - [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { - const float xi = float(data_a[row*p.KX + col]); - sum[tid].x += xi; - sum[tid].y += xi * xi; + // Input offset mirrors rms_norm.comp's stride-based scheme. + uint32_t a_offset = samp*stride_sample + channel*stride_channel + row*stride_row + get_aoffset(); + uint32_t b_offset = get_boffset(); + // Bias (C) is required to be aligned (misalign==0) by the fusion gate. + uint32_t c_offset = 0; + // Output is contiguous per row (CLIP layernorm dst); flat_row equals + // (samp*nchannels + channel)*nrows + row by construction. + uint32_t d_offset = flat_row*ncols + get_doffset(); + + vec2 sum = vec2(0.0f, 0.0f); + + [[unroll]] for (uint col = tid; col < ncols; col += BLOCK_SIZE) { + const float xi = float(data_a[a_offset + col]); + sum.x += xi; + sum.y += xi * xi; } + sumsh[tid] = sum; // sum up partial sums and write back result barrier(); [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) { if (tid < s) { - sum[tid] += sum[tid + s]; + sum += sumsh[tid + s]; + sumsh[tid] = sum; } barrier(); } + sum = sumsh[0]; - const float mean = sum[0].x / p.KX; - const float var = sum[0].y / p.KX - mean * mean; + const float mean = sum.x / float(ncols); + const float var = sum.y / float(ncols) - mean * mean; const float inv_std = inversesqrt(var + p.param1); - [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) { - data_d[row*p.KX + col] = D_TYPE((float(data_a[row*p.KX + col]) - mean) * inv_std); + [[unroll]] for (uint col = tid; col < ncols; col += BLOCK_SIZE) { + float result = (float(data_a[a_offset + col]) - mean) * inv_std; + if (do_multiply) { + result *= float(data_b[b_offset + col]); + } + if (do_add) { + result += float(data_c[c_offset + col]); + } + data_d[d_offset + col] = D_TYPE(result); } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index c94322496301..9569627cdd2d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -994,7 +994,7 @@ void process_shaders() { string_to_spv("mul_mat_vec_nc_f16_f32", "mul_mat_vec_nc.comp", {{"A_TYPE", "float16_t"}, {"A_TYPEV4", "f16vec4"}, {"B_TYPE", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}}); // Norms - string_to_spv("norm_f32", "norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); + string_to_spv("norm_f32", "norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("rms_norm_f32", "rms_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("rms_norm_partials_f32", "rms_norm_partials.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}})); diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index fba8a2315b44..db97f95ffe05 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -2723,6 +2723,35 @@ struct clip_model_loader { static void warmup(clip_ctx & ctx_clip, const clip_image_f32_batch & batch) { support_info_graph info; + // Disable FA on GPU projectors that lack efficient (coopmat) flash attention. + // Without coopmat, Vulkan uses FA_SCALAR which is ~2.6x slower than the matmul path + // for CLIP encoder attention (Mali-G715: 38 vs ~100 GFLOPS/s). Coopmat-capable GPUs + // keep FA enabled. Resolved at runtime via proc_address — no compile-time backend dep. + // Acts on AUTO *and* ENABLED (the addon enables FA by default) — an inefficient + // scalar-FA GPU should never be forced into FA; only explicit DISABLED is left alone. + // Default when the backend can't confirm efficient FA = KEEP it. Only ggml-vulkan + // implements the query (returning false for Mali/non-coopmat); backends that don't + // answer (Metal, CUDA, …) have efficient FA and must keep it — disabling there forces + // explicit attention whose QK^T overflows the clip compute buffer at high n_pos + // (GGML_ASSERT in ggml-backend, e.g. image_tile_mode=disabled + large image_max_tokens). + if (ctx_clip.flash_attn_type != CLIP_FLASH_ATTN_TYPE_DISABLED && + ctx_clip.backend && ctx_clip.backend != ctx_clip.backend_cpu) { + bool efficient_fa = true; + ggml_backend_dev_t dev = ggml_backend_get_device(ctx_clip.backend); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + if (reg) { + typedef bool (*supports_efficient_fa_t)(ggml_backend_t); + auto fn = (supports_efficient_fa_t)ggml_backend_reg_get_proc_address( + reg, "ggml_backend_supports_efficient_fa"); + if (fn) { + efficient_fa = fn(ctx_clip.backend); + } + } + if (!efficient_fa) { + ctx_clip.flash_attn_type = CLIP_FLASH_ATTN_TYPE_DISABLED; + } + } + if (ctx_clip.flash_attn_type == CLIP_FLASH_ATTN_TYPE_AUTO) { // Probe flash-attention support by forcing it on for the warmup // graph, then restore AUTO so the per-image budget heuristic in