Gemma4 moe pr v3

.github/workflows/build-openvino.yml

@@ -80,14 +80,14 @@ jobs:
         id: cmake_test_cpu
         run: |
           cd ${{ github.workspace }}
-          ctest --test-dir build/ReleaseOV -L main --verbose --timeout 2000
+          ctest --test-dir build/ReleaseOV -L main --verbose --timeout 4000
 
       - name: Test (GPU)
         id: cmake_test_gpu
         run: |
           cd ${{ github.workspace }}
           export GGML_OPENVINO_DEVICE=GPU
-          ctest --test-dir build/ReleaseOV -L main --verbose --timeout 3000
+          ctest --test-dir build/ReleaseOV -L main --verbose --output-on-failure --timeout 8000
 
   openvino-windows-2022:
     runs-on: windows-2022
@@ -163,4 +163,4 @@ jobs:
           call "%OPENVINO_ROOT%\setupvars.bat"
 
           cd build
-          ctest --test-dir ReleaseOV -L main -C Release --verbose --timeout 3000
+          ctest --test-dir ReleaseOV -L main -C Release --verbose --timeout 4000

ggml/src/ggml-openvino/ggml-decoder.cpp

@@ -565,7 +565,7 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
         if (node->op == GGML_OP_GATED_DELTA_NET) {
             model_params.state_size = node->src[0]->ne[0];
         }
-        if (node->op == GGML_OP_SCALE && is_kvcache(node->view_src, nullptr)) {
+        if (node->op == GGML_OP_SCALE && node->view_src != nullptr && is_kvcache(node->view_src, nullptr)) {
             compute_params.cache_rs_reset_len = ggml_nelements(node) / node->view_src->ne[0];
             compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->ne[0];
         }
@@ -958,6 +958,30 @@ std::shared_ptr<ov::Node> GgmlOvDecoder::create_weight_node(ggml_tensor * tensor
         return weight_node;
     }
 
+    // 3D quantized MoE expert weights [k, m, n_expert]: flatten to a rank-2
+    // [n_expert, m*k] tensor and build the dequant subgraph with use_bias=true (the
+    // exact f16 zero-point form). This is the path hit by test-backend-ops and the
+    // host-buffer load; the backend-buffer path builds the same node in set_tensor.
+    // translate_mul_mat_id gathers experts on axis 0 of this node and splits m*k.
+    if (ggml_is_quantized(tensor->type) && tensor->ne[2] > 1) {
+        GGML_ASSERT(tensor->ne[3] == 1 && "4D quantized expert weights are not supported");
+        GGML_ASSERT(ggml_is_contiguous(tensor) && "expert weights must be contiguous to flatten");
+        const int64_t n_expert = tensor->ne[2];
+        const int64_t m = tensor->ne[1];
+        const int64_t k = tensor->ne[0];
+        ggml_tensor flat_tensor = *tensor;
+        flat_tensor.ne[0] = m * k;
+        flat_tensor.ne[1] = n_expert;
+        flat_tensor.ne[2] = 1;
+        flat_tensor.ne[3] = 1;
+        flat_tensor.nb[1] = ggml_row_size(tensor->type, m * k);
+        flat_tensor.nb[2] = ggml_nbytes(tensor);
+        flat_tensor.nb[3] = ggml_nbytes(tensor);
+        OvWeight flat_weight = process_weight_tensor(&flat_tensor, tensor->data, nullptr, /*use_bias=*/true);
+        flat_weight.weight_node->set_friendly_name(tensor->name);
+        return flat_weight.weight_node;
+    }
+
     // There are three cases where we need to create a new weight node:
     // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor
     // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used
@@ -1673,8 +1697,22 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
         case GGML_OP_DIAG:
         case GGML_OP_TRI:
         case GGML_OP_REPEAT:
+        // Shape-preserving elementwise ops: the dynamic dim is unchanged from src[0].
+        // DIV/CLAMP are used in the MoE routing-weight normalization
+        // (sum_rows -> clamp -> div). If they are left untracked here the dynamic
+        // (token) dim is lost there, the captured prefill token count gets baked into
+        // the downstream reshapes, and every decoder layer after layer 0 turns static
+        // (which then triggers the GPU in-place-concat KV-cache corruption).
+        case GGML_OP_DIV:
+        case GGML_OP_CLAMP:
             m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]];
             break;
+        case GGML_OP_SUM_ROWS:
+            // SUM_ROWS reduces ggml axis 0 to size 1 and preserves all other axes, so the
+            // dynamic dim is preserved unless it was axis 0 (then it is summed away).
+            m_node_dynamic_dims[node] =
+                (m_node_dynamic_dims[node->src[0]] == 0) ? -1 : m_node_dynamic_dims[node->src[0]];
+            break;
         case GGML_OP_MUL_MAT_ID:
         case GGML_OP_SOLVE_TRI:
             m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]];

ggml/src/ggml-openvino/ggml-openvino-extra.cpp

@@ -173,6 +173,28 @@ bool ggml_openvino_is_npu() {
     return ggml_openvino_get_device_config().is_npu;
 }
 
+// Latched true once a MUL_MAT_ID op is seen during op placement; see header. Plain
+// non-atomic bool: placement runs single-threaded before the multi-threaded compute
+// that reads it, and the flag only ever transitions false->true (idempotent).
+static bool g_has_moe_expert_weights = false;
+
+void ggml_openvino_note_moe_expert_weight() {
+    g_has_moe_expert_weights = true;
+}
+
+bool ggml_openvino_has_moe_expert_weights() {
+    return g_has_moe_expert_weights;
+}
+
+bool ggml_openvino_full_moe_enabled() {
+    // Keep the whole MoE on one OV submodel instead of fragmenting the graph at every
+    // MoE routing node. Auto-detected: a MoE model is recognized when the expert-routed
+    // matmul (MUL_MAT_ID) has been seen (see ggml_openvino_note_moe_expert_weight).
+    // Enabled on the dynamic-shape devices (CPU and GPU); NPU uses the static path and
+    // keeps the fragmented behavior for now.
+    return !ggml_openvino_is_npu() && ggml_openvino_has_moe_expert_weights();
+}
+
 // Get the remote context for the current device (returns empty optional for CPU)
 std::optional<ov::RemoteContext> ggml_openvino_get_remote_context() {
     return ggml_openvino_get_device_config().remote_context;
@@ -252,9 +274,12 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
         return layout;
     }
 
-    // Most quantized weights use the existing 2D extraction path. MXFP4 also
-    // appears as 3D expert weights for MUL_MAT_ID, so allow that type through.
-    if (tensor->type != GGML_TYPE_MXFP4 && (tensor->ne[2] != 1 || tensor->ne[3] != 1)) {
+    // Handle 2D weight tensors, and 3D MoE expert weights [k, m, n_expert] which
+    // are treated as a flattened 2D [n_expert*m, k] tensor (each row is quantized
+    // independently along k, so the block layout is identical when flattened). This
+    // covers both our quantized gemma4 experts (Q4_K/Q5_1) and MXFP4 3D experts,
+    // which are handled by the dedicated block just below.
+    if (tensor->ne[3] != 1) {
         return layout;
     }
 
@@ -390,6 +415,10 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
     // For symmetric quantization, no zp needed (weights stored as signed)
     if (layout.is_symmetric) {
         layout.zp_size = 0;
+    } else if (use_bias) {
+        // use_bias stores the zero-point/bias as F16 (2 bytes/block), not a packed
+        // integer. Must size the buffer accordingly so the extracted data fits in-place.
+        layout.zp_size = n_blocks * sizeof(uint16_t);
     } else {
         layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks;
     }

ggml/src/ggml-openvino/ggml-openvino-extra.h

@@ -99,6 +99,24 @@ int ggml_openvino_getenv_int(const char * var, int default_value = 0);
 // Check if running on NPU
 bool ggml_openvino_is_npu();
 
+// MoE detection. ggml_openvino_note_moe_expert_weight() latches a process-global flag
+// that ggml_openvino_has_moe_expert_weights() reports. It is called from supports_op()
+// the first time a GGML_OP_MUL_MAT_ID (the expert-routed matmul) is seen, which is the
+// defining op of a MoE model. The latch is set at op-placement time (not weight load):
+// the scheduler queries op placement before the expert weights are streamed in, and it
+// makes multiple placement passes, so the first pass that encounters MUL_MAT_ID sets the
+// flag and subsequent passes converge on the full-MoE layout. This lets the backend
+// recognize "this is a MoE model" without any architecture name.
+void ggml_openvino_note_moe_expert_weight();
+bool ggml_openvino_has_moe_expert_weights();
+
+// Whether to keep the whole MoE on one OV submodel instead of fragmenting at every
+// MoE node (see the per-node "force to CPU" gates). Auto-detected: ON when the model
+// has expert-routed matmuls (a MoE model) on the dynamic-shape devices (CPU/GPU),
+// OFF on NPU (static path). Keeping the whole MoE on OV is numerically correct on both
+// CPU and GPU and avoids the graph fragmentation that caused index corruption on GPU.
+bool ggml_openvino_full_moe_enabled();
+
 // Get requantization type for a tensor type (returns nullopt if no requant needed)
 std::optional<ExtraQuantType> ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false);