Skip to content

qwen35 moe #258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wine99 wants to merge 1 commit into
base: dev_backend_openvino
Choose a base branch
from
Draft

qwen35 moe #258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion ggml/src/ggml-openvino/ggml-decoder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -565,7 +565,7 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
if (node->op == GGML_OP_GATED_DELTA_NET) {
model_params.state_size = node->src[0]->ne[0];
}
if (node->op == GGML_OP_SCALE && is_kvcache(node->view_src, nullptr)) {
if (node->op == GGML_OP_SCALE && node->view_src != nullptr && is_kvcache(node->view_src, nullptr)) {
compute_params.cache_rs_reset_len = ggml_nelements(node) / node->view_src->ne[0];
compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->ne[0];
}
Expand Down Expand Up @@ -1673,8 +1673,22 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
case GGML_OP_DIAG:
case GGML_OP_TRI:
case GGML_OP_REPEAT:
// Shape-preserving elementwise ops: the dynamic dim is unchanged from src[0].
// DIV/CLAMP are used in the MoE routing-weight normalization
// (sum_rows -> clamp -> div). If they are left untracked here the dynamic
// (token) dim is lost there, the captured prefill token count gets baked into
// the downstream reshapes, and every decoder layer after layer 0 turns static
// (which then triggers the GPU in-place-concat KV-cache corruption).
case GGML_OP_DIV:
case GGML_OP_CLAMP:
m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]];
break;
case GGML_OP_SUM_ROWS:
// SUM_ROWS reduces ggml axis 0 to size 1 and preserves all other axes, so the
// dynamic dim is preserved unless it was axis 0 (then it is summed away).
m_node_dynamic_dims[node] =
(m_node_dynamic_dims[node->src[0]] == 0) ? -1 : m_node_dynamic_dims[node->src[0]];
break;
case GGML_OP_MUL_MAT_ID:
case GGML_OP_SOLVE_TRI:
m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]];
Expand Down
20 changes: 16 additions & 4 deletions ggml/src/ggml-openvino/ggml-openvino-extra.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,12 +252,19 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
return layout;
}

// Most quantized weights use the existing 2D extraction path. MXFP4 also
// appears as 3D expert weights for MUL_MAT_ID, so allow that type through.
if (tensor->type != GGML_TYPE_MXFP4 && (tensor->ne[2] != 1 || tensor->ne[3] != 1)) {
// Most quantized weights use the existing 2D extraction path. 3D expert weights for
// MUL_MAT_ID (MoE) are also supported, either as MXFP4 (packed, dedicated branch below) or via the
// generic sizing math below, which is shape-agnostic (based on total element count). Only reject 4D.
if (tensor->ne[3] != 1) {
return layout;
}

// 3D MoE expert weights that are not requantized (see below) always use the exact f16
// zero-point extraction (see extract_quantized_weights), which needs a wider zp slot than
// the packed integer zero point -- must be kept in sync with that function so the buffer
// sizing here matches what process_weight_tensor actually writes.
const bool for_gather_matmul = tensor->ne[2] > 1;

int64_t n_elements = ggml_nelements(tensor);
const size_t alignment = 64; // Good for SIMD

Expand Down Expand Up @@ -387,9 +394,14 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
// Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block.
int64_t n_blocks = n_elements / layout.weights_per_block;
layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t));
// For symmetric quantization, no zp needed (weights stored as signed)
// For symmetric quantization, no zp needed (weights stored as signed). Asymmetric
// for_gather_matmul (3D MoE expert) weights use an exact f16 zero point (see
// extract_quantized_weights/make_int8_weights/make_int4_weights), which needs one f16 per
// block instead of a packed u4/u8 integer zero point.
if (layout.is_symmetric) {
layout.zp_size = 0;
} else if (use_bias || for_gather_matmul) {
layout.zp_size = n_blocks * sizeof(uint16_t);
} else {
layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks;
}
Expand Down
47 changes: 34 additions & 13 deletions ggml/src/ggml-openvino/ggml-openvino.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,25 @@ static void * ggml_backend_openvino_buffer_get_base(ggml_backend_buffer_t buffer
return ctx->data;
}

// Quantized types whose 3D MoE expert weights are routed through GatherMatmul (see
// extract_quantized_weights / make_int4_weights / make_int8_weights in ggml-quants.cpp), so they don't
// need the naive per-token materialization mul_mat_id_requires_large_tmp() guards against.
static bool is_gather_matmul_moe_expert_type(ggml_type type) {
switch (type) {
case GGML_TYPE_MXFP4:
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q8_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q5_K:
case GGML_TYPE_Q6_K:
return true;
default:
return false;
}
}

static bool is_stateful_enabled() {
return ggml_openvino_getenv_int("GGML_OPENVINO_STATEFUL_EXECUTION") != 0;
}
Expand Down Expand Up @@ -235,9 +254,10 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer
bool is_weight_buffer = (buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
// Full tensor set: offset=0, full size, not a view
bool is_full_tensor_set = (offset == 0 && size == ggml_nbytes(tensor) && tensor->view_src == nullptr);
// 2D tensor (typical weight shape)
// 2D tensor (typical weight shape), or a 3D MoE expert weight tensor of a type routed through
// GatherMatmul (see is_gather_matmul_moe_expert_type).
bool is_2d = (tensor->ne[2] == 1 && tensor->ne[3] == 1);
bool is_supported_weight_shape = is_2d || tensor->type == GGML_TYPE_MXFP4;
bool is_supported_weight_shape = is_2d || (tensor->ne[3] == 1 && is_gather_matmul_moe_expert_type(tensor->type));

if (is_weight_buffer && is_full_tensor_set && is_supported_weight_shape) {
try {
Expand Down Expand Up @@ -460,8 +480,8 @@ static size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buff
GGML_UNUSED(buft);

// For quantized weight tensors, we need extra space for extracted data.
if (ggml_is_quantized(tensor->type) &&
((tensor->ne[2] == 1 && tensor->ne[3] == 1) || tensor->type == GGML_TYPE_MXFP4)) {
if (ggml_is_quantized(tensor->type) && ((tensor->ne[2] == 1 && tensor->ne[3] == 1) ||
(tensor->ne[3] == 1 && is_gather_matmul_moe_expert_type(tensor->type)))) {
ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor);
if (layout.total_size > 0) {
// GGML_LOG_DEBUG("%s: tensor %s needs %zu bytes (original %zu, extracted: weights=%zu scales=%zu zp=%zu)\n",
Expand Down Expand Up @@ -1072,7 +1092,7 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
return true;
}
if (mul_mat_id_requires_large_tmp(op) &&
!(op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_MXFP4)) {
!(op->src[0] != nullptr && is_gather_matmul_moe_expert_type(op->src[0]->type))) {
return true;
}
break;
Expand Down Expand Up @@ -1231,11 +1251,11 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
// GGML_LOG_WARN("OpenVINO backend does not support GLU op %s\n", ggml_glu_op_name(ggml_get_glu_op(op)));
return false;
}
if (has_view_op_input(op)) {
// GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n",
// ggml_glu_op_name(ggml_get_glu_op(op)));
return false;
}
// if (has_view_op_input(op)) {
// // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n",
// // ggml_glu_op_name(ggml_get_glu_op(op)));
// return false;
// }
if (op->src[1] == nullptr && op->src[0]->ne[0] % 2 != 0) {
// triggers bug in ov gpu
return false;
Expand Down Expand Up @@ -1269,9 +1289,10 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
// GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type));
return false;
}
const bool is_supported_3d_mxfp4_moe = op->op == GGML_OP_MUL_MAT_ID && i == 0 &&
src->type == GGML_TYPE_MXFP4;
if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_mxfp4_moe) {
const bool is_supported_3d_moe_expert = op->op == GGML_OP_MUL_MAT_ID && i == 0 &&
is_gather_matmul_moe_expert_type(src->type) &&
(src->type == GGML_TYPE_MXFP4 || src->ne[3] == 1);
if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_moe_expert) {
// GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n");
return false;
}
Expand Down
Loading
Loading