From 9b2ad27623bb87d3e9733ea8b338c2f1c1a4a010 Mon Sep 17 00:00:00 2001 From: "Yu, Zijun" Date: Fri, 17 Jul 2026 17:58:11 +0800 Subject: [PATCH] temp qwen35 moe --- ggml/src/ggml-openvino/ggml-decoder.cpp | 16 +- .../src/ggml-openvino/ggml-openvino-extra.cpp | 20 ++- ggml/src/ggml-openvino/ggml-openvino.cpp | 47 ++++-- ggml/src/ggml-openvino/ggml-quants.cpp | 150 +++++++++++++----- ggml/src/ggml-openvino/ggml-quants.h | 26 ++- .../openvino/op/gather_matmul.hpp | 43 +++++ .../ggml-openvino/openvino/op/get_rows.cpp | 60 ++++++- .../ggml-openvino/openvino/op/mul_mat_id.cpp | 79 ++++----- ggml/src/ggml-openvino/openvino/op/view.cpp | 120 ++++++++++++++ .../pass/mark_dequantization_subgraph.h | 44 +++++ .../openvino/translate_session.cpp | 15 +- ggml/src/ggml-openvino/openvino/utils.cpp | 33 +++- 12 files changed, 542 insertions(+), 111 deletions(-) create mode 100644 ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp create mode 100644 ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 16f059380759..7fc17b1467d1 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -565,7 +565,7 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr if (node->op == GGML_OP_GATED_DELTA_NET) { model_params.state_size = node->src[0]->ne[0]; } - if (node->op == GGML_OP_SCALE && is_kvcache(node->view_src, nullptr)) { + if (node->op == GGML_OP_SCALE && node->view_src != nullptr && is_kvcache(node->view_src, nullptr)) { compute_params.cache_rs_reset_len = ggml_nelements(node) / node->view_src->ne[0]; compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->ne[0]; } @@ -1673,8 +1673,22 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { case GGML_OP_DIAG: case GGML_OP_TRI: case GGML_OP_REPEAT: + // Shape-preserving elementwise ops: the dynamic dim is unchanged from src[0]. + // DIV/CLAMP are used in the MoE routing-weight normalization + // (sum_rows -> clamp -> div). If they are left untracked here the dynamic + // (token) dim is lost there, the captured prefill token count gets baked into + // the downstream reshapes, and every decoder layer after layer 0 turns static + // (which then triggers the GPU in-place-concat KV-cache corruption). + case GGML_OP_DIV: + case GGML_OP_CLAMP: m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; break; + case GGML_OP_SUM_ROWS: + // SUM_ROWS reduces ggml axis 0 to size 1 and preserves all other axes, so the + // dynamic dim is preserved unless it was axis 0 (then it is summed away). + m_node_dynamic_dims[node] = + (m_node_dynamic_dims[node->src[0]] == 0) ? -1 : m_node_dynamic_dims[node->src[0]]; + break; case GGML_OP_MUL_MAT_ID: case GGML_OP_SOLVE_TRI: m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]]; diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 18df24c77e64..d1c2ac51c98b 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -252,12 +252,19 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten return layout; } - // Most quantized weights use the existing 2D extraction path. MXFP4 also - // appears as 3D expert weights for MUL_MAT_ID, so allow that type through. - if (tensor->type != GGML_TYPE_MXFP4 && (tensor->ne[2] != 1 || tensor->ne[3] != 1)) { + // Most quantized weights use the existing 2D extraction path. 3D expert weights for + // MUL_MAT_ID (MoE) are also supported, either as MXFP4 (packed, dedicated branch below) or via the + // generic sizing math below, which is shape-agnostic (based on total element count). Only reject 4D. + if (tensor->ne[3] != 1) { return layout; } + // 3D MoE expert weights that are not requantized (see below) always use the exact f16 + // zero-point extraction (see extract_quantized_weights), which needs a wider zp slot than + // the packed integer zero point -- must be kept in sync with that function so the buffer + // sizing here matches what process_weight_tensor actually writes. + const bool for_gather_matmul = tensor->ne[2] > 1; + int64_t n_elements = ggml_nelements(tensor); const size_t alignment = 64; // Good for SIMD @@ -387,9 +394,14 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten // Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block. int64_t n_blocks = n_elements / layout.weights_per_block; layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t)); - // For symmetric quantization, no zp needed (weights stored as signed) + // For symmetric quantization, no zp needed (weights stored as signed). Asymmetric + // for_gather_matmul (3D MoE expert) weights use an exact f16 zero point (see + // extract_quantized_weights/make_int8_weights/make_int4_weights), which needs one f16 per + // block instead of a packed u4/u8 integer zero point. if (layout.is_symmetric) { layout.zp_size = 0; + } else if (use_bias || for_gather_matmul) { + layout.zp_size = n_blocks * sizeof(uint16_t); } else { layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks; } diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 0601afe0939b..62ba8769c311 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -146,6 +146,25 @@ static void * ggml_backend_openvino_buffer_get_base(ggml_backend_buffer_t buffer return ctx->data; } +// Quantized types whose 3D MoE expert weights are routed through GatherMatmul (see +// extract_quantized_weights / make_int4_weights / make_int8_weights in ggml-quants.cpp), so they don't +// need the naive per-token materialization mul_mat_id_requires_large_tmp() guards against. +static bool is_gather_matmul_moe_expert_type(ggml_type type) { + switch (type) { + case GGML_TYPE_MXFP4: + case GGML_TYPE_Q4_0: + case GGML_TYPE_Q4_1: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q8_0: + case GGML_TYPE_Q5_1: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + return true; + default: + return false; + } +} + static bool is_stateful_enabled() { return ggml_openvino_getenv_int("GGML_OPENVINO_STATEFUL_EXECUTION") != 0; } @@ -235,9 +254,10 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer bool is_weight_buffer = (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS); // Full tensor set: offset=0, full size, not a view bool is_full_tensor_set = (offset == 0 && size == ggml_nbytes(tensor) && tensor->view_src == nullptr); - // 2D tensor (typical weight shape) + // 2D tensor (typical weight shape), or a 3D MoE expert weight tensor of a type routed through + // GatherMatmul (see is_gather_matmul_moe_expert_type). bool is_2d = (tensor->ne[2] == 1 && tensor->ne[3] == 1); - bool is_supported_weight_shape = is_2d || tensor->type == GGML_TYPE_MXFP4; + bool is_supported_weight_shape = is_2d || (tensor->ne[3] == 1 && is_gather_matmul_moe_expert_type(tensor->type)); if (is_weight_buffer && is_full_tensor_set && is_supported_weight_shape) { try { @@ -460,8 +480,8 @@ static size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buff GGML_UNUSED(buft); // For quantized weight tensors, we need extra space for extracted data. - if (ggml_is_quantized(tensor->type) && - ((tensor->ne[2] == 1 && tensor->ne[3] == 1) || tensor->type == GGML_TYPE_MXFP4)) { + if (ggml_is_quantized(tensor->type) && ((tensor->ne[2] == 1 && tensor->ne[3] == 1) || + (tensor->ne[3] == 1 && is_gather_matmul_moe_expert_type(tensor->type)))) { ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor); if (layout.total_size > 0) { // GGML_LOG_DEBUG("%s: tensor %s needs %zu bytes (original %zu, extracted: weights=%zu scales=%zu zp=%zu)\n", @@ -1072,7 +1092,7 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { return true; } if (mul_mat_id_requires_large_tmp(op) && - !(op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_MXFP4)) { + !(op->src[0] != nullptr && is_gather_matmul_moe_expert_type(op->src[0]->type))) { return true; } break; @@ -1231,11 +1251,11 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con // GGML_LOG_WARN("OpenVINO backend does not support GLU op %s\n", ggml_glu_op_name(ggml_get_glu_op(op))); return false; } - if (has_view_op_input(op)) { - // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n", - // ggml_glu_op_name(ggml_get_glu_op(op))); - return false; - } + // if (has_view_op_input(op)) { + // // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n", + // // ggml_glu_op_name(ggml_get_glu_op(op))); + // return false; + // } if (op->src[1] == nullptr && op->src[0]->ne[0] % 2 != 0) { // triggers bug in ov gpu return false; @@ -1269,9 +1289,10 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con // GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type)); return false; } - const bool is_supported_3d_mxfp4_moe = op->op == GGML_OP_MUL_MAT_ID && i == 0 && - src->type == GGML_TYPE_MXFP4; - if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_mxfp4_moe) { + const bool is_supported_3d_moe_expert = op->op == GGML_OP_MUL_MAT_ID && i == 0 && + is_gather_matmul_moe_expert_type(src->type) && + (src->type == GGML_TYPE_MXFP4 || src->ne[3] == 1); + if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_moe_expert) { // GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n"); return false; } diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index d4e4d8f660b0..54467cea1b69 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -18,8 +18,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -504,22 +505,34 @@ void extract_q5_k_data(const ggml_tensor * tensor, // TODO Reorder for make_intX_weights +// If for_gather_matmul is true, weight may be N-D (e.g. 3D MoE expert weights [n_expert, rows, cols]). +// The dequantization chain below is built as usual but left in f16 (no final Convert to f32) -- +// ov::pass::MarkDequantization (registered in translate_session.cpp) marks the chain so it survives +// model-build-time ConstantFolding. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul directly +// on top of the resulting f16 chain. ov::Output make_int8_weights(ov::Tensor & weight, ov::Tensor & scales, ov::Tensor & zp, size_t group_size, - bool use_bias) { + bool use_bias, + bool for_gather_matmul) { ov::Shape orig_shape = weight.get_shape(); bool is_signed = (weight.get_element_type() == ov::element::i8); // Symmetric: signed weights, no ZP // Expand dimensions for scales and zp/bias auto scale_shape = scales.get_shape(); - ov::Shape packed_shape = {orig_shape[0], orig_shape[1] / group_size, group_size}; + // Group the innermost (last) dimension. For 2D weights [rows, cols] this yields + // [rows, cols/group_size, group_size]; for 3D MoE experts [n_expert, rows, cols] this yields + // [n_expert, rows, cols/group_size, group_size]. + ov::Shape packed_shape = orig_shape; + packed_shape.back() /= group_size; + packed_shape.push_back(group_size); + const size_t group_dim = packed_shape.size() - 2; - if (packed_shape[1] == 1) { + if (packed_shape[group_dim] == 1) { // Requantized channel-wise case - packed_shape.erase(packed_shape.begin() + 1); + packed_shape.erase(packed_shape.begin() + group_dim); } else { scale_shape.push_back(1); scales.set_shape(scale_shape); @@ -539,7 +552,8 @@ ov::Output make_int8_weights(ov::Tensor & weight, static_cast(weight.data()), nullptr); weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; auto weights_f16 = std::make_shared(weights_node, ov::element::f16); - result = std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY); + auto mul = std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY); + result = mul; } else { // Unsigned path auto weights_node = std::make_shared(ov::element::u8, packed_shape, @@ -548,11 +562,25 @@ ov::Output make_int8_weights(ov::Tensor & weight, auto weights_f16 = std::make_shared(weights_node, ov::element::f16); if (use_bias && zp.get_size() > 0) { - // Bias path: w * s + b (zp tensor holds f16 bias values) - auto bias_f16 = std::make_shared(zp); - auto w_s = - std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY); - result = std::make_shared(w_s, bias_f16, ov::op::AutoBroadcastType::NUMPY); + // Accurate dequant in the FUSABLE zero-point form: (w - zp) * s, where the zero + // point is an exact f16 value zp = -bias/scale (the zp tensor holds bias values + // coming in). Algebraically equal to w*s + bias, but unlike an Add(bias) graph this + // matches CompressedWeightsBlock's pattern (Constant->Convert->Subtract->Multiply), + // so for_gather_matmul weights still fuse into GatherMatmulCompressed. Also avoids + // the round(min/scale) error of an integer zero point. Convert bias -> zero-point IN + // PLACE in the (possibly buffer-backed) zp tensor to avoid a duplicate allocation. + auto * bias_zp_data = zp.data(); + const auto * scale_data = scales.data(); + const size_t n = zp.get_size(); + for (size_t i = 0; i < n; i++) { + float s = static_cast(scale_data[i]); + float b = static_cast(bias_zp_data[i]); + bias_zp_data[i] = ov::float16(s != 0.0f ? -b / s : 0.0f); + } + auto zero_point_f16 = std::make_shared(zp); + auto w_zp = + std::make_shared(weights_f16, zero_point_f16, ov::op::AutoBroadcastType::NUMPY); + result = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); } else { // Zero point path: (w - zp) * s auto zero_point = std::make_shared(zp); @@ -563,37 +591,49 @@ ov::Output make_int8_weights(ov::Tensor & weight, auto zero_point_f16 = std::make_shared(zero_point, ov::element::f16); auto w_zp = std::make_shared(weights_f16, zero_point_f16, ov::op::AutoBroadcastType::NUMPY); - result = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); + auto mul = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); + result = mul; } } - if (packed_shape.size() != 2) { + if (packed_shape.size() != orig_shape.size()) { // If not requantized channel-wise case, reshape back to original shape auto final_shape = std::make_shared(ov::element::i64, ov::Shape{orig_shape.size()}, orig_shape); - result = std::make_shared(result, final_shape, false); + auto reshaped = std::make_shared(result, final_shape, false); + result = reshaped; } + if (for_gather_matmul) { + return result; + } return std::make_shared(result, ov::element::f32); } +// See make_int8_weights for the meaning of for_gather_matmul. ov::Output make_int4_weights(ov::Tensor & weight, ov::Tensor & scales, ov::Tensor & zp, size_t group_size, - bool use_bias) { + bool use_bias, + bool for_gather_matmul) { ov::Shape orig_weight_shape = weight.get_shape(); bool is_signed = (weight.get_element_type() == ov::element::i4); // Symmetric: signed weights, no ZP // Expand dimensions for scales and zp/bias ov::Shape scale_shape = scales.get_shape(); - // Create INT4 weight tensor - ov::Shape packed_shape = {orig_weight_shape[0], orig_weight_shape[1] / group_size, group_size}; + // Create INT4 weight tensor. Group the innermost (last) dimension: for 2D weights + // [rows, cols] this yields [rows, cols/group_size, group_size]; for 3D MoE experts + // [n_expert, rows, cols] this yields [n_expert, rows, cols/group_size, group_size]. + ov::Shape packed_shape = orig_weight_shape; + packed_shape.back() /= group_size; + packed_shape.push_back(group_size); + const size_t group_dim = packed_shape.size() - 2; - if (packed_shape[1] == 1) { + if (packed_shape[group_dim] == 1) { // Requantized channel-wise case - packed_shape.erase(packed_shape.begin() + 1); + packed_shape.erase(packed_shape.begin() + group_dim); } else { scale_shape.push_back(1); scales.set_shape(scale_shape); @@ -613,7 +653,8 @@ ov::Output make_int4_weights(ov::Tensor & weight, static_cast(weight.data()), nullptr); weights_node->get_rt_info()["__gguf_tensor_holder"] = weight; auto weights_f16 = std::make_shared(weights_node, ov::element::f16); - result = std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY); + auto mul = std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY); + result = mul; } else { // Unsigned path auto weights_node = std::make_shared(ov::element::u4, packed_shape, @@ -622,11 +663,23 @@ ov::Output make_int4_weights(ov::Tensor & weight, auto weights_f16 = std::make_shared(weights_node, ov::element::f16); if (use_bias && zp.get_size() > 0) { - // Bias path: w * s + b (zp tensor holds f16 bias values) - auto bias_f16 = std::make_shared(zp); - auto w_s = - std::make_shared(weights_f16, scales_f16, ov::op::AutoBroadcastType::NUMPY); - result = std::make_shared(w_s, bias_f16, ov::op::AutoBroadcastType::NUMPY); + // Accurate dequant in the FUSABLE zero-point form: (w - zp) * s with an exact f16 + // zp = -bias/scale. Equivalent to w*s + bias but matches CompressedWeightsBlock's + // pattern so for_gather_matmul weights still fuse into GatherMatmulCompressed, and + // avoids the round(min/scale) error of an integer zp. Convert bias -> zero-point IN + // PLACE in the (possibly buffer-backed) zp tensor to avoid a duplicate allocation. + auto * bias_zp_data = zp.data(); + const auto * scale_data = scales.data(); + const size_t n = zp.get_size(); + for (size_t i = 0; i < n; i++) { + float s = static_cast(scale_data[i]); + float b = static_cast(bias_zp_data[i]); + bias_zp_data[i] = ov::float16(s != 0.0f ? -b / s : 0.0f); + } + auto zero_points_f16 = std::make_shared(zp); + auto w_zp = + std::make_shared(weights_f16, zero_points_f16, ov::op::AutoBroadcastType::NUMPY); + result = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); } else { // Zero point path: (w - zp) * s auto zero_points_node = std::make_shared(zp); @@ -637,17 +690,22 @@ ov::Output make_int4_weights(ov::Tensor & weight, auto zero_points_f16 = std::make_shared(zero_points_node, ov::element::f16); auto w_zp = std::make_shared(weights_f16, zero_points_f16, ov::op::AutoBroadcastType::NUMPY); - result = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); + auto mul = std::make_shared(w_zp, scales_f16, ov::op::AutoBroadcastType::NUMPY); + result = mul; } } - if (packed_shape.size() != 2) { + if (packed_shape.size() != orig_weight_shape.size()) { // If not requantized channel-wise case, reshape back to original shape auto final_shape = std::make_shared(ov::element::i64, ov::Shape{orig_weight_shape.size()}, orig_weight_shape); - result = std::make_shared(result, final_shape, false); + auto reshaped = std::make_shared(result, final_shape, false); + result = reshaped; } + if (for_gather_matmul) { + return result; + } return std::make_shared(result, ov::element::f32); } @@ -730,6 +788,13 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, std::string(ggml_type_name(tensor->type))); } + // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point extraction + // (see make_int8_weights/make_int4_weights) rather than the rounded integer zero point -- + // round(min/scale) error is what corrupts Q4_K/Q5_1 experts, and the f16-zp form still fuses + // into GatherMatmulCompressed since it stays a Subtract, not an Add. + const bool for_gather_matmul = tensor->ne[2] > 1; + use_bias = use_bias || for_gather_matmul; + // Extract quantized data switch (tensor->type) { case GGML_TYPE_Q4_0: @@ -757,12 +822,13 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, throw std::runtime_error("Unsupported quantized type: " + std::string(ggml_type_name(tensor->type))); } - // Create the OpenVINO weight subgraph + // Create the OpenVINO weight subgraph. 3D expert weights (MoE) are routed through the + // GatherMatmul-oriented path: dequantized in f16, with constant folding disabled on the chain. ov::Output weight_node; if (is_u4) { - weight_node = make_int4_weights(weights, scales, zp, weights_per_block, use_bias); + weight_node = make_int4_weights(weights, scales, zp, weights_per_block, use_bias, for_gather_matmul); } else { - weight_node = make_int8_weights(weights, scales, zp, weights_per_block, use_bias); + weight_node = make_int8_weights(weights, scales, zp, weights_per_block, use_bias, for_gather_matmul); } auto result = weight_node.get_node_shared_ptr(); @@ -822,8 +888,11 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo OvWeight result; - // Get 2D shape for weights [rows, cols] - ov::Shape node_shape = {static_cast(tensor->ne[1]), static_cast(tensor->ne[0])}; + // Get shape for weights: [rows, cols], or [n_expert, rows, cols] for 3D MoE expert weights. + ov::Shape node_shape = (tensor->ne[2] > 1) ? + ov::Shape{static_cast(tensor->ne[2]), static_cast(tensor->ne[1]), + static_cast(tensor->ne[0])} : + ov::Shape{static_cast(tensor->ne[1]), static_cast(tensor->ne[0])}; // Handle F16/F32/BF16 weights if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) { @@ -865,6 +934,14 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo OPENVINO_THROW("Unsupported quantized type: ", ggml_type_name(tensor->type)); } + // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point path (see + // extract_quantized_weights) -- must be kept in sync with the "use_bias || for_gather_matmul" + // check in ggml_openvino_get_extracted_layout, which sizes/offsets the zp slot accordingly. + // Requantized tensors (layout.is_requant) are handled by requantize_to_buffers instead, whose + // zp sizing/type is unaffected by for_gather_matmul, so they are excluded here. + const bool for_gather_matmul = tensor->ne[2] > 1; + const bool zp_is_f16 = !layout.is_requant && (use_bias || for_gather_matmul); + const bool is_3d_mxfp4_moe = tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1); if (is_3d_mxfp4_moe) { ov::Shape packed_shape = {static_cast(tensor->ne[3]), @@ -914,7 +991,8 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo ov::element::f4e2m1 : (layout.is_symmetric ? (layout.is_u4 ? ov::element::i4 : ov::element::i8) : (layout.is_u4 ? ov::element::u4 : ov::element::u8)); - ov::Shape scale_shape = {node_shape[0], node_shape[1] / layout.weights_per_block}; + ov::Shape scale_shape = node_shape; + scale_shape.back() /= layout.weights_per_block; if (tensor->type == GGML_TYPE_MXFP4) { if (tensor->ne[2] == 1 && tensor->ne[3] == 1) { @@ -936,7 +1014,7 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; result.scales = ov::Tensor(scale_type, scale_shape, buf_base + layout.scales_offset); if (!layout.is_symmetric) { - ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; + ov::element::Type zp_type = zp_is_f16 ? ov::element::f16 : (layout.is_u4 ? ov::element::u4 : ov::element::u8); result.zp = ov::Tensor(zp_type, scale_shape, buf_base + layout.zp_offset); } // else: result.zp remains default-constructed (empty) for symmetric @@ -945,7 +1023,7 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; result.scales = ov::Tensor(scale_type, scale_shape); if (!layout.is_symmetric) { - if (use_bias) { + if (zp_is_f16) { result.zp = ov::Tensor(ov::element::f16, scale_shape); } else { ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; diff --git a/ggml/src/ggml-openvino/ggml-quants.h b/ggml/src/ggml-openvino/ggml-quants.h index 1b89fd887e16..ca5faf66a609 100644 --- a/ggml/src/ggml-openvino/ggml-quants.h +++ b/ggml/src/ggml-openvino/ggml-quants.h @@ -54,17 +54,30 @@ void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov static constexpr size_t GGML_QUANTIZATION_GROUP_SIZE = 32; +// If for_gather_matmul is true, the weight tensor may be N-D (e.g. 3D MoE expert weights +// [n_expert, rows, cols]). The dequantization chain (Convert->[Subtract]->Multiply) is built as +// usual but left in f16 (no final Convert to f32) -- ov::pass::MarkDequantization (registered in +// translate_session.cpp) marks the chain so it survives model-build-time ConstantFolding -- see +// make_int8_weights.cpp/make_int4_weights.cpp. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul +// directly from the resulting f16 dequant chain. +// +// When use_bias is true (explicitly, or implicitly because for_gather_matmul is true), the zp +// tensor is expected to hold an exact f16 bias value (rather than a rounded integer zero point); +// it is converted in place into an exact zero_point = -bias/scale and consumed via Subtract, not +// Add, so the chain still matches OpenVINO's Convert->Subtract->Multiply decompression pattern. ov::Output make_int8_weights(ov::Tensor & weight, ov::Tensor & scales, ov::Tensor & zp, size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, - bool use_bias = false); + bool use_bias = false, + bool for_gather_matmul = false); ov::Output make_int4_weights(ov::Tensor & weight, ov::Tensor & scales, ov::Tensor & zp, size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, - bool use_bias = false); + bool use_bias = false, + bool for_gather_matmul = false); ov::Output make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales); @@ -80,7 +93,9 @@ std::shared_ptr extract_quantized_weights( ov::Tensor & weights, ov::Tensor & scales, ov::Tensor & zp, - bool use_bias = false); // Use fp bias instead of quantized zero_point (for test-backend-ops) + bool use_bias = false); // Use an exact f16 zero point (vs. a rounded integer one); always + // used for for_gather_matmul (3D MoE expert) weights regardless of + // this flag, and also settable explicitly for test-backend-ops. // Requantize weights from tensor to target format, writing to provided buffers // For F16 target, only weights buffer is used (scales/zp ignored) @@ -133,7 +148,10 @@ OvWeight process_weight_tensor( const ggml_tensor * tensor, const void * data, // Source data pointer (may differ from tensor->data) void * output_base_ptr = nullptr, // Base pointer for output buffers (or nullptr for internal allocation) - bool use_bias = false); // Use fp bias instead of quantized zero_point, only used in test-backend-ops + bool use_bias = false); // Use an exact f16 zero point (vs. a rounded integer one); + // always used for for_gather_matmul (3D MoE expert) weights + // regardless of this flag, and also settable explicitly for + // test-backend-ops. void quantize_q4_0(const float * x, ov::Tensor & weights_arr, diff --git a/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp b/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp new file mode 100644 index 000000000000..39bd744b0c86 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp @@ -0,0 +1,43 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +// Local mirror of OpenVINO's internal ov::op::internal::GatherMatmul op. +// +// The op class body (validate_and_infer_types / clone_with_new_inputs) is +// provided by the linked libopenvino.so; only the declaration is needed here so +// the backend can construct the node directly (same approach as GatedDeltaNet). +// The class layout must stay in sync with +// openvino/src/common/transformations/include/ov_ops/gather_matmul.hpp +// +// \note GatherMatmul op class is under development and subject to change. + +#pragma once + +#include "openvino/op/op.hpp" + +namespace ov::op::internal { + +class OPENVINO_API GatherMatmul : public ov::op::Op { +public: + OPENVINO_OP("GatherMatmul") + + GatherMatmul() = default; + + GatherMatmul(const ov::Output& A, + const ov::Output& B, + const ov::Output& indices, + const ov::Output& bias); + + GatherMatmul(const ov::Output& A, const ov::Output& B, const ov::Output& indices); + + std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override; + + void validate_and_infer_types() override; + +private: + // the weights matrix B is expected to have the transposed form [group, N, K] + static constexpr bool transp_a = false; + static constexpr bool transp_b = true; +}; + +} // namespace ov::op::internal diff --git a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp index 3d5c8cb413d0..2ac8ec0ba1df 100644 --- a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp @@ -5,9 +5,12 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include @@ -59,7 +62,62 @@ OutputVector translate_get_rows(const NodeContext & context) { auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1}); data = std::make_shared(data, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); - res = std::make_shared(data, indices, axis, 1); + // data: [batch, rows, ...], indices: [batch, n] - this is a batched gather + // (batch_dims=1) along the rows axis. The data and indices batch dims are + // logically equal (both == n_tokens) but reach this node through independent + // reshapes, so the GPU plugin's gather shape inference cannot prove + // data.shape[0] == indices.shape[0] and rejects the node. We must tie both + // batch dims to the SAME value, and crucially that value must stay DYNAMIC. + const auto data_ps = data.get_partial_shape(); + const auto idx_ps = indices.get_partial_shape(); + const bool data_batch_static = data_ps.rank().is_static() && data_ps[0].is_static(); + const bool idx_batch_dynamic = idx_ps.rank().is_dynamic() || idx_ps[0].is_dynamic(); + + if (data_batch_static && idx_batch_dynamic) { + // MoE per-expert-scale path: `data` is a statically-tiled REPEAT + // (ggml_repeat_4d(scale, 1, n_expert, n_tokens, 1)) whose batch dim is a + // compile-time-constant n_tokens, and every batch slice is IDENTICAL (it was + // tiled from a single [1, n_expert, 1] scale). `indices` (selected_experts) + // carries the genuinely dynamic token dim. Broadcasting indices up to the + // static data batch (the naive fix) would freeze the token dim to the + // captured prefill length, and that static value then flows through the + // gather into the residual stream, making every following decoder layer + // static -> triggers the GPU in-place-concat KV-cache corruption (only + // layer 0 stays dynamic). A static->dynamic Broadcast cannot expand, so + // instead collapse the redundant data batch to 1 and broadcast 1->dynamic to + // match the indices batch. Mathematically identical (the slices are equal), + // and the whole graph stays dynamic. + auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axis0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto data_b1 = std::make_shared(data, zero, one, one, axis0); // [1, rows, ...] + + auto idx_shape = std::make_shared(indices, ov::element::i64); + auto idx_batch = get_dimensions(idx_shape, {0}); // [batch] (dynamic) + auto data_b1_shape = std::make_shared(data_b1, ov::element::i64); + const auto rank = data_ps.rank().get_length(); + std::vector rest_axes; + for (int a = 1; a < rank; ++a) { + rest_axes.push_back(a); + } + auto data_rest = get_dimensions(data_b1_shape, rest_axes); // [rows, ...] + auto data_target = std::make_shared(ov::OutputVector{idx_batch, data_rest}, 0); + data = + std::make_shared(data_b1, data_target, ov::op::BroadcastType::BIDIRECTIONAL); + res = std::make_shared(data, indices, axis, 1); + } else { + // General case: tie the indices batch to the data batch (the data batch is + // already dynamic, e.g. the routing-weights gather whose data comes from the + // activations). Broadcast indices to [data_batch, indices_n]. + auto data_shape = std::make_shared(data, ov::element::i64); + auto data_batch = get_dimensions(data_shape, {0}); // [batch] + auto idx_shape = std::make_shared(indices, ov::element::i64); + auto idx_n = get_dimensions(idx_shape, {1}); // [n] + auto idx_target = std::make_shared(ov::OutputVector{data_batch, idx_n}, 0); + indices = std::make_shared(indices, idx_target, + ov::op::BroadcastType::BIDIRECTIONAL); + res = std::make_shared(data, indices, axis, 1); + } } } else if (context.is_stateful() && data.get_partial_shape().rank() == 3) { auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1}); diff --git a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp index 6df2784c2e45..80f430d320a2 100644 --- a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp @@ -1,6 +1,7 @@ #include "../node_context.h" #include "../op_table.h" #include "../utils.h" +#include "gather_matmul.hpp" #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #include #include @@ -144,22 +146,36 @@ OutputVector translate_mul_mat_id(const NodeContext & context) { context.get_name()); } + // General (non-packed) path: dense F32/F16/BF16 weights, or the f16 dequantization chain for + // quantized MoE experts (see extract_quantized_weights / make_int4_weights / make_int8_weights in + // ggml-quants.cpp). Routed through ov::op::internal::GatherMatmul instead of a naive + // Gather+Broadcast+MatMul, so the selected expert's full weight matrix is never materialized per + // token. The CPU plugin's ConvertGatherMatmulToGatherMatmulCompressed pass (run during + // compile_model) fuses the dequantization chain feeding GatherMatmul's B input into a + // GatherMatmulCompressed node automatically, as long as MarkDequantization has marked the chain -- + // see translate_session.cpp's apply_transformations for the MarkDequantization registration. + // // OpenVINO sees GGML tensors in reversed dimension order: - // weights: [1, n_expert, m, k] // activations: [1, n_tokens, n_used_or_1, k] // ids: [1, 1, n_tokens, n_used] - // Rebuild the logical ranks explicitly from the 4D inputs instead of relying - // on fixed squeeze axes: real graphs can arrive through VIEW/RESHAPE chains - // where singleton axes are still represented differently at this point. - auto expert_weights_shape_4d = std::make_shared(expert_weights, ov::element::i64); + // expert_weights is either [1, n_expert, m, k] (4D, e.g. non-quantized weights without a + // pre-built extra) or already [n_expert, m, k] (3D, weights routed through + // process_weight_tensor) -- GatherMatmul's B input expects the latter. + auto expert_weights_rank = expert_weights.get_partial_shape().rank(); + FRONT_END_OP_CONVERSION_CHECK(expert_weights_rank.is_static(), + "Expected static rank for MUL_MAT_ID expert weights"); + if (expert_weights_rank.get_length() == 4) { + auto expert_weights_shape_4d = std::make_shared(expert_weights, ov::element::i64); + auto expert_weights_shape_3d = get_dimensions(expert_weights_shape_4d, {1, 2, 3}); + expert_weights = std::make_shared(expert_weights, expert_weights_shape_3d, false); + } + auto activations_shape_4d = std::make_shared(activations, ov::element::i64); auto ids_shape_4d = std::make_shared(ids, ov::element::i64); - auto expert_weights_shape_3d = get_dimensions(expert_weights_shape_4d, {1, 2, 3}); auto activations_shape_3d = get_dimensions(activations_shape_4d, {1, 2, 3}); auto ids_shape_2d = get_dimensions(ids_shape_4d, {2, 3}); - expert_weights = std::make_shared(expert_weights, expert_weights_shape_3d, false); activations = std::make_shared(activations, activations_shape_3d, false); ids = std::make_shared(ids, ids_shape_2d, false); @@ -167,51 +183,24 @@ OutputVector translate_mul_mat_id(const NodeContext & context) { ids = std::make_shared(ids, ov::element::i32); } - auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); - ov::Output selected_weights = std::make_shared(expert_weights, ids, gather_axis); - const auto output_type = context.get_output_type(); - if (selected_weights.get_element_type() != ov::element::f32) { - selected_weights = std::make_shared(selected_weights, ov::element::f32); - } if (activations.get_element_type() != ov::element::f32) { activations = std::make_shared(activations, ov::element::f32); } - auto activations_shape = std::make_shared(activations, ov::element::i64); - auto ids_shape = std::make_shared(ids, ov::element::i64); - ov::Output acts_target_dims = std::make_shared( - ov::OutputVector{ - get_dimensions(activations_shape, {0}), - get_dimensions(ids_shape, {1}), - get_dimensions(activations_shape, {2}), - }, - 0); - ov::Output acts_broadcasted = - std::make_shared(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL); - - auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); - auto activations_expanded = std::make_shared(acts_broadcasted, unsqueeze_axes); + // GatherMatmul's A input is [n_used_or_1, n_tokens, k]; activations_3d is + // [n_tokens, n_used_or_1, k]. + auto activations_transpose_order = const_i64({1, 0, 2}); + ov::Output activations_for_gather = + std::make_shared(activations, activations_transpose_order); - auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto output_shape = context.get_output_shape(); - FRONT_END_OP_CONVERSION_CHECK(output_shape.rank().is_static() && output_shape.rank().get_length() == 4, - "Unexpected MUL_MAT_ID output rank"); - FRONT_END_OP_CONVERSION_CHECK(output_shape[3].is_static(), "Expected static row dimension for MUL_MAT_ID output"); - const auto row_dim_value = output_shape[3].get_length(); - auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {row_dim_value}); + ov::Output result = std::make_shared(activations_for_gather, expert_weights, ids); - ov::Output result = - std::make_shared(activations_expanded, selected_weights, false, true); - - auto result_target_dims = std::make_shared( - ov::OutputVector{ - batch_dim, - get_dimensions(ids_shape, {0, 1}), - row_dim, - }, - 0); - result = std::make_shared(result, result_target_dims, false); + // result is [n_used, n_tokens, m]; GGML expects [1, n_tokens, n_used, m]. + auto result_transpose_order = const_i64({1, 0, 2}); + result = std::make_shared(result, result_transpose_order); + auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + result = std::make_shared(result, unsqueeze_axes); if (result.get_element_type() != output_type) { result = std::make_shared(result, output_type); diff --git a/ggml/src/ggml-openvino/openvino/op/view.cpp b/ggml/src/ggml-openvino/openvino/op/view.cpp index 28004dcd2d8d..138526cb49c6 100644 --- a/ggml/src/ggml-openvino/openvino/op/view.cpp +++ b/ggml/src/ggml-openvino/openvino/op/view.cpp @@ -1,8 +1,11 @@ #include "../op_table.h" #include "../utils.h" +#include #include +#include #include +#include #include #include @@ -15,6 +18,123 @@ OutputVector translate_view(const NodeContext & context) { num_inputs_check(context, 1, 1); if (!context.is_static()) { + // On the stateless/non-static path VIEW is normally a no-op (consumers re-slice). + // EXCEPTION: the MoE expert aggregation slices each expert plane out of + // ffn_moe_weighted [n_embd, n_expert_used, n_tokens] with ggml_view_2d and then + // sums the planes with a chain of ADDs (llama-graph.cpp). Those ADDs read this + // VIEW node directly from the tensor map and do NOT re-slice, so a no-op here + // makes every plane the full tensor and the expert sum collapses. Materialize the + // single-expert slice here. Gated by name (ffn_moe_weighted...view) so it can't + // affect any other view. + const std::string & vname = context.get_name(); + if (vname.find("ffn_moe_weighted") != std::string::npos) { + auto src_ps = context.get_input_shape(0); + auto dst_ps = context.get_output_shape(); + if (src_ps.rank().is_static() && dst_ps.rank().is_static() && src_ps.rank() == dst_ps.rank() && + src_ps.is_static() && dst_ps.is_static()) { + auto sst = context.get_input_stride(0); + auto dst = context.get_output_stride(); + size_t voff = context.get_output_op_offset(); + auto ss = src_ps.to_shape(); + auto dd = dst_ps.to_shape(); + const size_t nd = ss.size(); + if (sst.size() == nd && dst.size() == nd) { + // Map each dst axis of size>1 to a src axis with equal (size,stride); + // the unmatched src axis of size>1 is the indexed expert axis. + // dst_to_src[d] records which src axis each dst axis came from, so we can + // later pull the dynamic (token) dim from the right source axis at runtime. + std::vector used(nd, false); + std::vector dst_to_src(nd, -1); + bool ok = true; + for (size_t d = 0; d < nd; ++d) { + if (dd[d] == 1) { + continue; + } + int found = -1; + for (size_t s = 0; s < nd; ++s) { + if (!used[s] && ss[s] == dd[d] && sst[s] == dst[d]) { + found = (int) s; + break; + } + } + if (found < 0) { + ok = false; + break; + } + used[found] = true; + dst_to_src[d] = found; + } + int dropped = -1; + if (ok) { + for (size_t s = 0; s < nd; ++s) { + if (!used[s] && ss[s] > 1) { + if (dropped >= 0) { + ok = false; + break; + } + dropped = (int) s; + } + } + } + if (ok && dropped >= 0) { + const size_t dstr = sst[dropped]; + const int64_t dsz = (int64_t) ss[dropped]; + if (dstr > 0 && voff % dstr == 0) { + const int64_t sel = (int64_t) (voff / dstr); + if (sel >= 0 && sel < dsz) { + ov::Output sl = std::make_shared( + context.get_input(0), + ov::op::v0::Constant::create(ov::element::i64, {1}, {sel}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {sel + 1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {dropped})); + // Build the reshape target from the (concrete) dst shape, but + // keep the dynamic token axis dynamic instead of freezing it + // to the captured n_tokens. Without this the constant dst + // shape bakes in the prefill token count and the static value + // flows downstream, turning every later decoder layer static + // (the GPU in-place-concat KV-cache bug). The token axis is + // PERMUTED between the sliced input and the dst (e.g. input + // [1,tok,expert,emb] -> dst [1,1,tok,emb]), so special_zero + // (which copies the same-position dim) is not enough: pull the + // dynamic dim from the correct SOURCE axis via ShapeOf+Gather + // and place it at the dst token position. + const int32_t dyn = context.get_op_dynamic_dim(); // output ggml axis, -1 if none + int dst_ov_axis = (dyn != -1) ? (3 - (int) dyn) : -1; // get_shape() reverses ggml order + int src_ov_axis = (dst_ov_axis >= 0 && dst_ov_axis < (int) nd) + ? dst_to_src[dst_ov_axis] + : -1; + if (dst_ov_axis >= 0 && src_ov_axis >= 0) { + // target = concat of per-axis scalars; the token axis is a + // runtime Gather of the slice's shape, the rest are constants. + auto sl_shape = std::make_shared(sl, ov::element::i64); + auto tok_dim = std::make_shared( + sl_shape, + ov::op::v0::Constant::create(ov::element::i64, {1}, {src_ov_axis}), + ov::op::v0::Constant::create(ov::element::i64, {}, {0})); + ov::OutputVector parts; + for (int a = 0; a < (int) nd; ++a) { + if (a == dst_ov_axis) { + parts.push_back(tok_dim); + } else { + parts.push_back(ov::op::v0::Constant::create( + ov::element::i64, {1}, {(int64_t) dd[a]})); + } + } + auto dc = std::make_shared(parts, 0); + auto rs = std::make_shared(sl, dc, false); + return rename_outputs_with_suffix({rs}, context.get_name()); + } + auto dc = ov::op::v0::Constant::create( + ov::element::i64, {nd}, std::vector(dd.begin(), dd.end())); + auto rs = std::make_shared(sl, dc, false); + return rename_outputs_with_suffix({rs}, context.get_name()); + } + } + } + } + } + } return {context.get_input(0)}; } diff --git a/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h b/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h new file mode 100644 index 000000000000..3cb8fce845e1 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h @@ -0,0 +1,44 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +// Local mirror of OpenVINO's ov::pass::MarkDequantization pass declaration. +// +// The pass body is provided by the linked libopenvino.so; only the declaration is needed here so +// we can register it directly in our own TranslateSession::apply_transformations (same approach as +// MarkCompressedFloatConstants's local mirror in mark_decompression_convert_constant_folding.h). This +// lets us mark our GatherMatmul dequantization chain with disable_constant_folding regardless of the +// CPU/GPU plugin's own is_decompression_multiply() consumer allowlist. +// The class layout must stay in sync with +// openvino/src/common/transformations/include/transformations/low_precision/mark_dequantization_subgraph.hpp + +#pragma once + +#include "openvino/core/type/element_type.hpp" +#include "openvino/core/visibility.hpp" +#include "openvino/pass/matcher_pass.hpp" + +#ifdef OPENVINO_STATIC_LIBRARY +# define TRANSFORMATIONS_API +#else +# ifdef IMPLEMENT_OPENVINO_API +# define TRANSFORMATIONS_API OPENVINO_CORE_EXPORTS +# else +# define TRANSFORMATIONS_API OPENVINO_CORE_IMPORTS +# endif // IMPLEMENT_OPENVINO_API +#endif // OPENVINO_STATIC_LIBRARY + +namespace ov { +namespace pass { + +class TRANSFORMATIONS_API MarkDequantization; + +} // namespace pass +} // namespace ov + +class ov::pass::MarkDequantization : public MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("MarkDequantization"); + explicit MarkDequantization(const element::TypeVector & precisions, + bool fold_subtract_const = false, + bool fold_multiply_const = true); +}; diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index b0cc142a86d7..0b72bcb78e65 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -4,6 +4,7 @@ #include "ggml-openvino/openvino/utils.h" #include "input_model.h" #include "pass/mark_decompression_convert_constant_folding.h" +#include "pass/mark_dequantization_subgraph.h" #include "pass/squeeze_matmul.h" #include "rt_info/weightless_caching_attributes.hpp" @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -320,10 +322,13 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo // // Small constants (< 16 elements) are excluded since they may be introduced by // optimization patterns and the overhead is negligible. + // + // Note: use shape_size() rather than byte_size()/element_type().size() - GatherMatmul's default + // bias is a Constant(element::dynamic, Shape{0}), whose element_type().size() is 0 and would + // divide by zero. size_t offset = 0; for (auto & node : resulting_model->get_ordered_ops()) { - if (auto cnst = ov::as_type_ptr(node); - cnst && cnst->get_byte_size() / cnst->get_element_type().size() >= 16) { + if (auto cnst = ov::as_type_ptr(node); cnst && ov::shape_size(cnst->get_shape()) >= 16) { auto & rt_info = cnst->get_rt_info(); if (rt_info.find(ov::WeightlessCacheAttribute::get_type_info_static()) == rt_info.end()) { rt_info[ov::WeightlessCacheAttribute::get_type_info_static()] = @@ -340,6 +345,12 @@ std::shared_ptr TranslateSession::apply_transformations(std::shared_ptr(); + // Marks the Convert/Subtract/Multiply nodes of our GatherMatmul dequantization chain + // (make_int4_weights/make_int8_weights, for_gather_matmul=true) with disable_constant_folding, + // so it survives ConstantFolding regardless of whether the target plugin's own + // is_decompression_multiply() recognizes GatherMatmul as a valid consumer. + manager.register_pass( + std::vector{ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4}); if (ggml_model_decoder->is_stateful()) { const auto kv_param_res_names = ggml_model_decoder->get_kv_param_res_names(); diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index e622d2305047..a99b9c839764 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -292,17 +292,40 @@ ov::Output process_view_input_new(const NodeContext & context, int inp // If translate_view already resolved this VIEW (produced a Slice), the input // will already have the expected shape — skip re-slicing. + // + // Two notions of "matches" are accepted per axis: + // - both dims static and equal, OR + // - both dims dynamic. + // The dynamic case matters for the MoE expert-plane views: translate_view now emits a + // DYNAMIC-token slice (so the token dim is not frozen). An all-static-only check would + // see the dynamic token dim, decide the shapes "don't match", and fall through to + // re-slice/flatten the already-resolved view (a Reshape to the full flattened + // n_expert_used*n_embd tail, which then conflicts with the single-plane input). Treat a + // dynamic-vs-dynamic axis as matching so the already-resolved view is reused as-is. + // + // A third case matters for split-model MoE fragments: translate_view resolves the + // expert-plane view against the fragment's INPUT parameter. When the graph is split + // the token axis of that parameter may already be concrete (static n_tokens) even + // though get_view_input_ov_shape() still reports it as dynamic (-1). The resolved + // view is then static [1,1,n_tokens,n_embd] while `expected` is [1,1,?,n_embd]. + // An "expected dynamic, actual static" axis is a valid concretization of the SAME + // resolved view, so treat it as matching too. Falling through to process_single_view + // here would re-slice/re-flatten the already-resolved single-plane view against the + // recorded (multi-plane) source strides and emit a constant-target Reshape whose baked + // dims no longer divide the concretized input -> "dimensions do not evenly divide". auto expected_ov_shape = context.get_view_input_ov_shape(input_index, 0); auto actual_shape = input.get_partial_shape(); if (expected_ov_shape.rank().is_static() && actual_shape.rank().is_static() && expected_ov_shape.rank() == actual_shape.rank()) { bool shapes_match = true; for (int64_t i = 0; i < expected_ov_shape.rank().get_length(); ++i) { - if (!expected_ov_shape[i].is_static() || !actual_shape[i].is_static()) { - shapes_match = false; - break; - } - if (expected_ov_shape[i] != actual_shape[i]) { + const bool both_dynamic = expected_ov_shape[i].is_dynamic() && actual_shape[i].is_dynamic(); + const bool both_static_equal = expected_ov_shape[i].is_static() && actual_shape[i].is_static() && + expected_ov_shape[i] == actual_shape[i]; + // expected dynamic, actual static: the resolved view already carries the + // concrete size for this fragment; reuse it rather than re-materializing. + const bool expected_dyn_actual_static = expected_ov_shape[i].is_dynamic() && actual_shape[i].is_static(); + if (!both_dynamic && !both_static_equal && !expected_dyn_actual_static) { shapes_match = false; break; }