ggml-ve: run F16 weights as BF16 + compile the VEBP/Q4K N>1 prompt graph (ggml-org#72)

VEBP keeps token_embd as F16 (tied, so it's both the embedding and the lm_head).
The VE has no F16 path, so GET_ROWS + that MUL_MAT were refused and the graph
fragmented. Two changes make VEBP fully self-contained and let its prompt graph
compile:

1. F16 -> BF16 on HBM upload (same 2-byte size, strides unchanged), served by
   the existing BF16 GET_ROWS / matvec / colmajor paths. get_rows/mul_mat
   supports + dispatch accept F16; the graph compiler maps src_type F16->BF16
   and uploads the converted copy. Conversion uses the F16C row helpers in 1M
   chunks (a per-element call loop was ~10 s for the 621M-elem token_embd,
   charged to the first prompt eval).

2. Removed a stale guard that refused Q4_K/VEBP MUL_MAT when ne[1] != 1. It
   dated from when N>1 was refused entirely; now the codegen loops the matvec
   _inner over the n_tok columns, so it handles N>1. The guard was rejecting
   EVERY VEBP prompt graph (Qcur is VEBP, ne[1]=N) -> the prompt silently ran
   on the interpreter. (A new first-execute verbose log made this visible.)

Also fixed N>1 codegen vectorization found while profiling (the .L showed it was
NOT a table overflow — it was unvectorized loops): the MUL `src1[e % period]`
modulo forced scalar code -> restored the nested broadcast form; added `restrict`
to the element-wise / RMS_NORM / GLU pointers (void*-cast aliasing).

Measured (GGML_VE_HBM=1, -fa on, -ctk/-ctv bf16, warm, run twice):
  Llama-3.2-3B  prompt 57 -> 65,  decode 48 -> 56   (vectorization fixes)
  Bonsai-VEBP   prompt 9.8 -> 12.65 (1.29x, V.OP 12% -> 92%),  decode 33.5 -> 38.7
All outputs token-for-token identical to the interpreter.

VEBP's prompt gain is smaller than Llama's 16x because its per-token ternary
matvec is compute-bound (V.OP already 92%), so the fork/join fusion saves a
smaller fraction. Further gains need a batched ternary matmul (read the weight
once across N columns) — a follow-up.

Author: wmeddie

ggml/src/ggml-ve/common.hpp

@@ -89,6 +89,9 @@ enum ggml_ve_hbm_format {
     GGML_VE_HBM_VEBP_WS       = 7,  // VEBP interleaved sign plane  [blk][word][256]
     GGML_VE_HBM_VEBP_WN       = 8,  // VEBP interleaved nonzero plane
     GGML_VE_HBM_VEBP_WSCALE   = 9,  // VEBP interleaved group scales [blk][grp][256]
+    GGML_VE_HBM_BF16_FROM_F16 = 10, // F16 weight converted to BF16 on upload
+                                    // (VE has no F16 path; BF16 is the same
+                                    //  2-byte size, so strides are unchanged)
 };
 
 #endif // GGML_VE_COMMON_HPP

ggml/src/ggml-ve/ggml-ve.cpp

@@ -419,6 +419,10 @@ ggml_status backend_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph)
                 if (cg2) {
                     new_entry.cg         = cg2;
                     new_entry.executable = gc.execute(cg2, ctx, cgraph);
+                    if (gc_verbose) {
+                        fprintf(stderr, "[VE-GC] first execute sig=%016lx n_nodes=%d -> executable=%d\n",
+                                (unsigned long) sig, cgraph->n_nodes, (int) new_entry.executable);
+                    }
                 } else if (gc_verbose) {
                     fprintf(stderr, "[VE-GC] compile failed for sig=%016lx\n", (unsigned long) sig);
                 }

ggml/src/ggml-ve/graph_compiler.cpp

@@ -229,7 +229,12 @@ bool GraphCompiler::trace_one(ggml_tensor * node) {
         }
         out_idx = assign_buffer(src, out_kind);
         for (int i = 0; i < 4; ++i) out_ne[i] = src->ne[i];
-        if (slot == 0) op.src_type = src->type;
+        // F16 weights are uploaded as BF16 (the VE has no F16 path); map the
+        // type here so MUL_MAT classification and the GET_ROWS codegen take the
+        // BF16 path, matching the BF16-converted HBM copy uploaded at execute.
+        if (slot == 0) {
+            op.src_type = (src->type == GGML_TYPE_F16) ? GGML_TYPE_BF16 : src->type;
+        }
     };
     fill_src(0, node->src[0], op.src0_idx, op.src0_kind, op.src0_ne);
 
@@ -309,12 +314,11 @@ bool GraphCompiler::trace_one(ggml_tensor * node) {
             else if (op.src_type == GGML_TYPE_Q4_K) op.type = OpType::MUL_MAT_Q4K;
             else if (op.src_type == GGML_TYPE_VEBP) op.type = OpType::MUL_MAT_VEBP;
             else                                     op.type = OpType::MUL_MAT_F32;
-            // Q4_K / VEBP inners are matvec-only (N=1). The compiler already
-            // refuses N>1 MUL_MAT above (prompt eval), but guard explicitly.
-            if ((op.type == OpType::MUL_MAT_Q4K || op.type == OpType::MUL_MAT_VEBP)
-                && node->src[1] && node->src[1]->ne[1] != 1) {
-                return false;
-            }
+            // N>1 prompt eval is handled by looping the matvec _inner over the
+            // n_tok activation columns (see gen_op_code MUL_MAT_VEBP/Q4K), so
+            // Q4_K / VEBP MUL_MAT no longer needs to be matvec-only. (Removing
+            // this guard is what lets the VEBP/Q4K prompt graph compile instead
+            // of being exiled to the interpreter.)
             break;
         case GGML_OP_ROPE: {
             int32_t mode = 0, n_dims = 0;
@@ -805,8 +809,8 @@ std::string GraphCompiler::gen_op_code(const TracedOp & op, int idx) const {
                 ss << "        float eps = " << flit(op.p.rms_norm.eps) << ";\n";
                 ss << "        #pragma omp for\n";
                 ss << "        for (long r = 0; r < rows; r++) {\n";
-                ss << "            const float* x = (const float*)" << src0 << " + r * cols;\n";
-                ss << "            float* y = (float*)" << dst << " + r * cols;\n";
+                ss << "            const float * restrict x = (const float*)" << src0 << " + r * cols;\n";
+                ss << "            float * restrict y = (float*)" << dst << " + r * cols;\n";
                 ss << "            float sumsq = 0.f;\n";
                 ss << "            for (int j = 0; j < cols; j++) sumsq += x[j] * x[j];\n";
                 ss << "            float inv = 1.f / sqrtf(sumsq / cols + eps);\n";
@@ -822,8 +826,8 @@ std::string GraphCompiler::gen_op_code(const TracedOp & op, int idx) const {
                 ss << "        float eps = " << flit(op.p.rms_norm.eps) << ";\n";
                 ss << "        #pragma omp for\n";
                 ss << "        for (long r = 0; r < rows; r++) {\n";
-                ss << "            const float* x = (const float*)" << src0 << " + r * cols;\n";
-                ss << "            float* y = (float*)" << dst << " + r * cols;\n";
+                ss << "            const float * restrict x = (const float*)" << src0 << " + r * cols;\n";
+                ss << "            float * restrict y = (float*)" << dst << " + r * cols;\n";
                 ss << "            float sumsq = 0.f;\n";
                 ss << "            for (int j = 0; j < cols; j++) sumsq += x[j] * x[j];\n";
                 ss << "            float inv = 1.f / sqrtf(sumsq / cols + eps);\n";
@@ -870,22 +874,38 @@ std::string GraphCompiler::gen_op_code(const TracedOp & op, int idx) const {
             int64_t src1_period = op.src1_ne[0] * op.src1_ne[1] * op.src1_ne[2] * op.src1_ne[3];
             if (src1_period == 0) src1_period = op.src1_ne[0];
             if (src1_period <= 0) src1_period = 1;
-            ss << "    #pragma omp for\n";
-            ss << "    for (int64_t e = 0; e < " << elem_n << "; e++) {\n";
-            ss << "        ((float*)" << dst << ")[e] = "
-               << "((float*)" << src0 << ")[e] * "
-               << "((float*)" << src1 << ")[e % " << src1_period << "LL];\n";
+            // Nested (broadcast-block) form, NOT `src1[e % period]`: a modulo in
+            // the inner loop trips NCC's "loop division overhead" and forces
+            // scalar code. The inner loop runs the (compile-time-constant)
+            // period with contiguous src1[i], so it vectorises; the outer loop
+            // (reps = total/period, runtime) is shared across the team.
+            // `restrict` kills the void*-cast aliasing ("Dependency unknown").
+            ss << "    {\n";
+            ss << "        float * restrict yv = (float*)" << dst << ";\n";
+            ss << "        const float * restrict av = (const float*)" << src0 << ";\n";
+            ss << "        const float * restrict bv = (const float*)" << src1 << ";\n";
+            ss << "        const int64_t period = " << src1_period << "LL;\n";
+            ss << "        const int64_t reps   = (" << elem_n << ") / period;\n";
+            ss << "        #pragma omp for\n";
+            ss << "        for (int64_t r = 0; r < reps; r++) {\n";
+            ss << "            const float * restrict ar = av + r*period;\n";
+            ss << "            float * restrict yr = yv + r*period;\n";
+            ss << "            for (int64_t i = 0; i < period; i++) yr[i] = ar[i] * bv[i];\n";
+            ss << "        }\n";
             ss << "    }\n";
             break;
         }
 
         case OpType::ADD:
             // src0 and src1 are the same shape (residual + branch). Element-wise
             // over the full per-token range (pt*n_tok when scaling, else full).
-            ss << "    #pragma omp for\n";
-            ss << "    for (int64_t e = 0; e < " << elem_n << "; e++) {\n";
-            ss << "        ((float*)" << dst << ")[e] = "
-               << "((float*)" << src0 << ")[e] + ((float*)" << src1 << ")[e];\n";
+            // restrict pointers so NCC vectorises (no assumed aliasing).
+            ss << "    {\n";
+            ss << "        float * restrict yv = (float*)" << dst << ";\n";
+            ss << "        const float * restrict av = (const float*)" << src0 << ";\n";
+            ss << "        const float * restrict bv = (const float*)" << src1 << ";\n";
+            ss << "        #pragma omp for\n";
+            ss << "        for (int64_t e = 0; e < " << elem_n << "; e++) yv[e] = av[e] + bv[e];\n";
             ss << "    }\n";
             break;
 
@@ -1222,9 +1242,9 @@ std::string GraphCompiler::gen_op_code(const TracedOp & op, int idx) const {
             // size-independent. Clamp before expf — the VE's vectorised expf
             // returns NaN past |x|~88 (see CLAUDE.md).
             ss << "    {\n";
-            ss << "        float* y    = (float*)" << dst  << ";\n";
-            ss << "        float* gate = (float*)" << src0 << ";\n";
-            ss << "        float* up   = (float*)" << src1 << ";\n";
+            ss << "        float * restrict y    = (float*)" << dst  << ";\n";
+            ss << "        const float * restrict gate = (const float*)" << src0 << ";\n";
+            ss << "        const float * restrict up   = (const float*)" << src1 << ";\n";
             ss << "        long total = (long)(" << elem_n << ");\n";
             ss << "        #pragma omp for\n";
             ss << "        for (long i = 0; i < total; i++) {\n";
@@ -1790,9 +1810,15 @@ bool GraphCompiler::execute(CompiledGraph * graph,
                 // 875-node decode graph was refused over this one weight.
                 size_t nb = ggml_nbytes(c);
                 const char * nm = (c->name && c->name[0]) ? c->name : nullptr;
-                VEDAdeviceptr w_hbm = nm
-                    ? bctx->cache().get_or_upload_by_name(nm, c->data, nb)
-                    : bctx->cache().get_or_upload(c->data, nb);
+                // F16 weights (e.g. a tied/F16 token_embd) have no VE F16 path:
+                // convert to BF16 once on upload (same byte size, so the view
+                // offset below is unchanged). The codegen treats this slot as
+                // BF16 (src_type is mapped F16->BF16 at trace).
+                VEDAdeviceptr w_hbm = (c->type == GGML_TYPE_F16)
+                    ? bctx->cache().get_or_upload_f16_as_bf16(nm, c->data, nb)
+                    : (nm
+                        ? bctx->cache().get_or_upload_by_name(nm, c->data, nb)
+                        : bctx->cache().get_or_upload(c->data, nb));
                 if (w_hbm) {
                     size_t off = (const uint8_t *) src_for_addr->data - (const uint8_t *) c->data;
                     hbm = w_hbm + off;

ggml/src/ggml-ve/hbm_cache.hpp

@@ -64,7 +64,8 @@ class hbm_weight_cache {
         for (auto & e : entries_) {
             if (e.host_data == host_data && e.size == size &&
                 e.format != GGML_VE_HBM_FP32_COLMAJOR &&
-                e.format != GGML_VE_HBM_BF16_COLMAJOR) {
+                e.format != GGML_VE_HBM_BF16_COLMAJOR &&
+                e.format != GGML_VE_HBM_BF16_FROM_F16) {
                 hits_++;
                 return e.vptr;
             }
@@ -101,6 +102,56 @@ class hbm_weight_cache {
         return v;
     }
 
+    // F16 weight -> BF16 on upload. The VE has no F16 vector path, so we
+    // re-pack each F16 element to BF16 (both are 2 bytes, so the byte size and
+    // every row stride are identical) and serve it through the normal BF16
+    // kernels — this is what lets VEBP's F16 token_embd run GET_ROWS and the
+    // tied lm_head MUL_MAT on the VE instead of fragmenting onto the CPU.
+    // Keyed by name with a distinct format so it never aliases a raw entry.
+    VEDAdeviceptr get_or_upload_f16_as_bf16(const char * tensor_name,
+                                            const void * host_f16,
+                                            size_t size_bytes) {
+        if (tensor_name != nullptr) {
+            for (auto & e : entries_) {
+                if (e.format == GGML_VE_HBM_BF16_FROM_F16 &&
+                    e.name == tensor_name && e.size == size_bytes) {
+                    hits_++;
+                    return e.vptr;
+                }
+            }
+        }
+        for (auto & e : entries_) {
+            if (e.format == GGML_VE_HBM_BF16_FROM_F16 &&
+                e.host_data == host_f16 && e.size == size_bytes) {
+                hits_++;
+                return e.vptr;
+            }
+        }
+        const size_t n = size_bytes / sizeof(uint16_t);
+        std::vector<uint16_t> bf16(n);
+        const ggml_fp16_t * src = (const ggml_fp16_t *) host_f16;
+        // Convert via ggml's row helpers (F16C-accelerated F16->F32, then a
+        // shift+round F32->BF16) in chunks. A per-element ggml_fp16_to_fp32 /
+        // ggml_fp32_to_bf16 call loop is ~16 ns/elem (~10 s for a 621M-element
+        // token_embd) — slow enough to dominate the first prompt eval; the row
+        // helpers bring it to well under a second.
+        {
+            const size_t CHUNK = 1u << 20;  // 1M elems -> 4 MB f32 scratch
+            std::vector<float> f32(CHUNK < n ? CHUNK : n);
+            for (size_t off = 0; off < n; off += CHUNK) {
+                const size_t cnt = (n - off < CHUNK) ? (n - off) : CHUNK;
+                ggml_fp16_to_fp32_row(src + off, f32.data(), (int64_t) cnt);
+                ggml_fp32_to_bf16_row(f32.data(),
+                                      (ggml_bf16_t *) (bf16.data() + off),
+                                      (int64_t) cnt);
+            }
+        }
+        VEDAdeviceptr v = upload(bf16.data(), size_bytes);
+        if (v == 0) return 0;
+        record(v, size_bytes, host_f16, tensor_name, GGML_VE_HBM_BF16_FROM_F16);
+        return v;
+    }
+
     // Q4_K canonical-split upload WITH PRE-DECODED HEADERS.
     //
     // Pre-decoding moves the per-block scale work (h2f conversion of d/dmin,

ggml/src/ggml-ve/ops/get_rows.cpp

@@ -21,7 +21,10 @@ bool get_rows_supports(const ggml_tensor * op) {
     const ggml_tensor * src = op->src[0];  // embeddings
     const ggml_tensor * idx = op->src[1];  // i32 indices
     if (src == nullptr || idx == nullptr) return false;
-    if (src->type != GGML_TYPE_BF16 && src->type != GGML_TYPE_F32) return false;
+    // F16 src (e.g. a tied/F16 token_embd) is accepted: the VE has no F16 path,
+    // so it's converted to BF16 once at HBM upload and served by the BF16 kernel.
+    if (src->type != GGML_TYPE_BF16 && src->type != GGML_TYPE_F32 &&
+        src->type != GGML_TYPE_F16) return false;
     if (idx->type != GGML_TYPE_I32) return false;
     // Need contiguous indices and output.
     if (!ggml_is_contiguous(idx) || !ggml_is_contiguous(op)) return false;
@@ -37,8 +40,12 @@ bool get_rows(backend_context * ctx, ggml_tensor * dst) {
 
     // src (embedding table) is typically a CPU-side BF16 weight on first
     // call; resolve_in uploads it via hbm_weight_cache so subsequent calls
-    // re-use the cached HBM ptr.
-    const VEDAdeviceptr src_hbm = ctx->resolve_in(src);
+    // re-use the cached HBM ptr. F16 weights are converted to BF16 once on
+    // upload (same byte size) and served by the BF16 kernel below.
+    const VEDAdeviceptr src_hbm =
+        (src->type == GGML_TYPE_F16)
+            ? ctx->cache().get_or_upload_f16_as_bf16(src->name, src->data, ggml_nbytes(src))
+            : ctx->resolve_in(src);
     const VEDAdeviceptr dst_hbm = ctx->resolve_out(dst);
     if (src_hbm == 0 || dst_hbm == 0) return false;
 
@@ -72,7 +79,7 @@ bool get_rows(backend_context * ctx, ggml_tensor * dst) {
     }
     ctx->enqueue_hbm_free(idx_tmp);
 
-    kernel_id kid = (src->type == GGML_TYPE_BF16)
+    kernel_id kid = (src->type == GGML_TYPE_BF16 || src->type == GGML_TYPE_F16)
         ? K_GET_ROWS_BF16_F32_HBM_HBM
         : K_GET_ROWS_F32_F32_HBM_HBM;
     VEDAfunction fn = ctx->fn(kid);

ggml/src/ggml-ve/ops/mul_mat.cpp

@@ -38,7 +38,11 @@ bool mul_mat_supports(const ggml_tensor * op) {
     const ggml_tensor * x = op->src[1];
     if (w == nullptr || x == nullptr) return rej("missing srcs");
     if (x->type != GGML_TYPE_F32) return rej("x not f32");
-    if (w->type != GGML_TYPE_F32 && w->type != GGML_TYPE_BF16) return rej("w type");
+    // F16 weights (e.g. a tied/F16 token_embd used as the lm_head) are accepted:
+    // the VE has no F16 path, so they're converted to BF16 once at HBM upload
+    // (same byte size) and run through the BF16 matvec/colmajor paths below.
+    if (w->type != GGML_TYPE_F32 && w->type != GGML_TYPE_BF16 &&
+        w->type != GGML_TYPE_F16) return rej("w type");
 
     // Shapes: dst = src0 @ src1^T  ->  K matches, M = src0 row dim, N = src1 row dim.
     const int64_t K = w->ne[0];
@@ -62,9 +66,9 @@ bool mul_mat_supports(const ggml_tensor * op) {
     //   BF16  N>1  : ve_bf16_matmul_hbm_full     (sgemv_packed_bf16_unr per batch)
     //    F32  N=1  : ve_f32_matvec_hbm_full
     //    F32  N>1  : ve_f32_sgemm_batched_cblas_hbm  (NLC cblas_sgemm)
-    if (w->type == GGML_TYPE_BF16 && (K % 16) != 0) {
+    if ((w->type == GGML_TYPE_BF16 || w->type == GGML_TYPE_F16) && (K % 16) != 0) {
         // The packed BF16 sgemv unrolls 16 across K, so K must be a multiple
-        // of 16 to stay inside the row.
+        // of 16 to stay inside the row. (F16 is converted to BF16, same rule.)
         return rej("BF16 K%16 != 0");
     }
     if (M <= 0 || K <= 0 || N <= 0) return rej("zero dim");
@@ -108,7 +112,13 @@ bool mul_mat(backend_context * ctx, ggml_tensor * dst) {
     const uint64_t K = (uint64_t) w->ne[0];
     const uint64_t N = (uint64_t) x->ne[1];
 
-    const VEDAdeviceptr w_vptr = ctx->resolve_in(w);
+    // F16 weight -> BF16 once on upload (same byte size); downstream treats it
+    // as BF16 (w_is_bf16). All other types go through the normal resolver.
+    const bool w_is_bf16 = (w->type == GGML_TYPE_BF16 || w->type == GGML_TYPE_F16);
+    const VEDAdeviceptr w_vptr =
+        (w->type == GGML_TYPE_F16)
+            ? ctx->cache().get_or_upload_f16_as_bf16(w->name, w->data, ggml_nbytes(w))
+            : ctx->resolve_in(w);
     const VEDAdeviceptr x_vptr = ctx->resolve_in(x);
     const VEDAdeviceptr y_vptr = ctx->resolve_out(dst);
     if (w_vptr == 0 || x_vptr == 0 || y_vptr == 0) return false;
@@ -149,7 +159,7 @@ bool mul_mat(backend_context * ctx, ggml_tensor * dst) {
         (std::getenv("GGML_VE_COLMAJOR_N1") != nullptr);
     if (colmajor_enabled
         && (N > 1 || n1_colmajor)
-        && w->type == GGML_TYPE_BF16
+        && w_is_bf16
         && ctx->dev() && ctx->dev()->colmajor
         && ctx->fn(K_BF16_TO_F32_COLMAJOR_HBM) != 0
         && ctx->fn(K_F32_SGEMM_BATCHED_CBLAS_HBM_NOTRANS) != 0) {
@@ -196,7 +206,7 @@ bool mul_mat(backend_context * ctx, ggml_tensor * dst) {
     VEDAfunction fn = 0;
     bool include_N = false;
 
-    if (w->type == GGML_TYPE_BF16) {
+    if (w_is_bf16) {
         if (N == 1) {
             fn = ctx->fn(K_BF16_MATVEC_HBM_FULL);
         } else {