Skip to content

Commit 3c264fa

Browse files
am17anam17an
committed
llama: Add option to merge gate and exp weights
1 parent a5bb8ba commit 3c264fa

11 files changed

Lines changed: 147 additions & 47 deletions

‎convert_hf_to_gguf.py‎

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,8 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path,
116116
split_max_tensors: int = 0, split_max_size: int = 0, dry_run: bool = False,
117117
small_first_shard: bool = False, hparams: dict[str, Any] | None = None, remote_hf_model_id: str | None = None,
118118
disable_mistral_community_chat_template: bool = False,
119-
sentence_transformers_dense_modules: bool = False):
119+
sentence_transformers_dense_modules: bool = False,
120+
fuse_gate_up_exps: bool = False):
120121
if type(self) is ModelBase or \
121122
type(self) is TextModel or \
122123
type(self) is MmprojModel:
@@ -135,6 +136,9 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path,
135136
self.dry_run = dry_run
136137
self.remote_hf_model_id = remote_hf_model_id
137138
self.sentence_transformers_dense_modules = sentence_transformers_dense_modules
139+
self.fuse_gate_up_exps = fuse_gate_up_exps
140+
self._gate_exp_buffer: dict[int, Tensor] = {}
141+
self._up_exp_buffer: dict[int, Tensor] = {}
138142
self.hparams = ModelBase.load_hparams(self.dir_model, self.is_mistral_format) if hparams is None else hparams
139143
self.model_tensors = self.index_tensors(remote_hf_model_id=remote_hf_model_id)
140144
self.metadata_override = metadata_override
@@ -514,8 +518,36 @@ def set_gguf_parameters(self):
514518
raise NotImplementedError("set_gguf_parameters() must be implemented in subclasses")
515519

516520
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
517-
del bid # unused
518-
return [(self.map_tensor_name(name), data_torch)]
521+
new_name = self.map_tensor_name(name)
522+
523+
# Handle gate/up expert tensor fusion if enabled
524+
if self.fuse_gate_up_exps and bid is not None:
525+
if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_GATE_EXP, bid):
526+
self._gate_exp_buffer[bid] = data_torch
527+
# Check if up_exps is already buffered for this layer
528+
if bid in self._up_exp_buffer:
529+
gate_data = self._gate_exp_buffer.pop(bid)
530+
up_data = self._up_exp_buffer.pop(bid)
531+
# gate/up shape: (n_expert, n_ff, n_embd), concatenate to (n_expert, n_ff*2, n_embd)
532+
fused_data = torch.cat([gate_data, up_data], dim=1)
533+
fused_name = f"blk.{bid}.ffn_gate_up_exps.weight"
534+
logger.info(f"Fused gate_exps and up_exps for layer {bid}")
535+
return [(fused_name, fused_data)]
536+
return [] # Wait for up_exps
537+
elif self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.FFN_UP_EXP, bid):
538+
self._up_exp_buffer[bid] = data_torch
539+
# Check if gate_exps is already buffered for this layer
540+
if bid in self._gate_exp_buffer:
541+
gate_data = self._gate_exp_buffer.pop(bid)
542+
up_data = self._up_exp_buffer.pop(bid)
543+
# gate/up shape: (n_expert, n_ff, n_embd), concatenate to (n_expert, n_ff*2, n_embd)
544+
fused_data = torch.cat([gate_data, up_data], dim=1)
545+
fused_name = f"blk.{bid}.ffn_gate_up_exps.weight"
546+
logger.info(f"Fused gate_exps and up_exps for layer {bid}")
547+
return [(fused_name, fused_data)]
548+
return [] # Wait for gate_exps
549+
550+
return [(new_name, data_torch)]
519551

520552
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
521553
del name, new_name, bid, n_dims # unused
@@ -11121,6 +11153,11 @@ def parse_args() -> argparse.Namespace:
1112111153
"Default these modules are not included.")
1112211154
)
1112311155

11156+
parser.add_argument(
11157+
"--fuse-gate-up-exps", action="store_true",
11158+
help="Fuse gate_exps and up_exps tensors into a single gate_up_exps tensor for MoE models.",
11159+
)
11160+
1112411161
args = parser.parse_args()
1112511162
if not args.print_supported_models and args.model is None:
1112611163
parser.error("the following arguments are required: model")
@@ -11258,7 +11295,8 @@ def main() -> None:
1125811295
split_max_size=split_str_to_n_bytes(args.split_max_size), dry_run=args.dry_run,
1125911296
small_first_shard=args.no_tensor_first_split,
1126011297
remote_hf_model_id=hf_repo_id, disable_mistral_community_chat_template=disable_mistral_community_chat_template,
11261-
sentence_transformers_dense_modules=args.sentence_transformers_dense_modules
11298+
sentence_transformers_dense_modules=args.sentence_transformers_dense_modules,
11299+
fuse_gate_up_exps=args.fuse_gate_up_exps
1126211300
)
1126311301

1126411302
if args.vocab_only:

‎gguf-py/gguf/constants.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,7 @@ class MODEL_TENSOR(IntEnum):
511511
FFN_GATE_EXP = auto()
512512
FFN_DOWN_EXP = auto()
513513
FFN_UP_EXP = auto()
514+
FFN_GATE_UP_EXP = auto()
514515
FFN_GATE_SHEXP = auto()
515516
FFN_DOWN_SHEXP = auto()
516517
FFN_UP_SHEXP = auto()
@@ -937,6 +938,7 @@ class MODEL_TENSOR(IntEnum):
937938
MODEL_TENSOR.FFN_GATE_EXP: "blk.{bid}.ffn_gate_exps",
938939
MODEL_TENSOR.FFN_DOWN_EXP: "blk.{bid}.ffn_down_exps",
939940
MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up_exps",
941+
MODEL_TENSOR.FFN_GATE_UP_EXP: "blk.{bid}.ffn_gate_up_exps",
940942
MODEL_TENSOR.FFN_EXP_PROBS_B: "blk.{bid}.exp_probs_b",
941943
MODEL_TENSOR.LAYER_OUT_NORM: "blk.{bid}.layer_output_norm",
942944
MODEL_TENSOR.PER_LAYER_TOKEN_EMBD: "per_layer_token_embd", # gemma3n
@@ -3115,6 +3117,7 @@ class MODEL_TENSOR(IntEnum):
31153117
MODEL_TENSOR.ATTN_OUT,
31163118
MODEL_TENSOR.ATTN_SINKS,
31173119
MODEL_TENSOR.FFN_GATE_INP,
3120+
MODEL_TENSOR.FFN_GATE_UP_EXP,
31183121
MODEL_TENSOR.FFN_GATE_EXP,
31193122
MODEL_TENSOR.FFN_DOWN_EXP,
31203123
MODEL_TENSOR.FFN_UP_EXP,

‎gguf-py/gguf/tensor_mapping.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -555,6 +555,10 @@ class TensorNameMap:
555555
"model.layers.{bid}.mlp.chunk_experts.gate_proj", # grovemoe
556556
),
557557

558+
MODEL_TENSOR.FFN_GATE_UP_EXP: (
559+
"model.layers.{bid}.mlp.experts.gate_up_proj", # gpt-oss
560+
),
561+
558562
# Feed-forward down
559563
MODEL_TENSOR.FFN_DOWN: (
560564
"gpt_neox.layers.{bid}.mlp.dense_4h_to_h", # gptneox

‎src/llama-arch.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,7 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
335335
{ LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" },
336336
{ LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },
337337
{ LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" },
338+
{ LLM_TENSOR_FFN_GATE_UP_EXPS, "blk.%d.ffn_gate_up_exps" },
338339
{ LLM_TENSOR_ATTN_POST_NORM, "blk.%d.post_attention_norm" },
339340
{ LLM_TENSOR_ATTN_Q_NORM, "blk.%d.attn_q_norm" },
340341
{ LLM_TENSOR_ATTN_K_NORM, "blk.%d.attn_k_norm" },
@@ -1497,6 +1498,7 @@ static std::set<llm_tensor> llm_get_tensor_names(llm_arch arch) {
14971498
LLM_TENSOR_FFN_GATE_INP,
14981499
LLM_TENSOR_FFN_GATE_EXPS,
14991500
LLM_TENSOR_FFN_DOWN_EXPS,
1501+
LLM_TENSOR_FFN_GATE_UP_EXPS,
15001502
LLM_TENSOR_FFN_UP_EXPS,
15011503
LLM_TENSOR_FFN_GATE_INP_SHEXP,
15021504
LLM_TENSOR_FFN_GATE_SHEXP,
@@ -2088,6 +2090,7 @@ static std::set<llm_tensor> llm_get_tensor_names(llm_arch arch) {
20882090
LLM_TENSOR_ATTN_OUT,
20892091
LLM_TENSOR_ATTN_SINKS,
20902092
LLM_TENSOR_FFN_GATE_INP,
2093+
LLM_TENSOR_FFN_GATE_UP_EXPS,
20912094
LLM_TENSOR_FFN_GATE_EXPS,
20922095
LLM_TENSOR_FFN_DOWN_EXPS,
20932096
LLM_TENSOR_FFN_UP_EXPS,
@@ -2434,6 +2437,7 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
24342437
{LLM_TENSOR_FFN_DOWN_EXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},
24352438
{LLM_TENSOR_FFN_GATE_EXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},
24362439
{LLM_TENSOR_FFN_UP_EXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},
2440+
{LLM_TENSOR_FFN_GATE_UP_EXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},
24372441
{LLM_TENSOR_FFN_DOWN_CHEXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},
24382442
{LLM_TENSOR_FFN_GATE_CHEXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},
24392443
{LLM_TENSOR_FFN_UP_CHEXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},

‎src/llama-arch.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,7 @@ enum llm_tensor {
357357
LLM_TENSOR_FFN_DOWN_EXPS, // merged experts
358358
LLM_TENSOR_FFN_GATE_EXPS,
359359
LLM_TENSOR_FFN_UP_EXPS,
360+
LLM_TENSOR_FFN_GATE_UP_EXPS,
360361
LLM_TENSOR_FFN_DOWN_SHEXP,
361362
LLM_TENSOR_FFN_GATE_SHEXP,
362363
LLM_TENSOR_FFN_UP_SHEXP,

‎src/llama-graph.cpp‎

Lines changed: 47 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1070,7 +1070,8 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
10701070
float w_scale,
10711071
llama_expert_gating_func_type gating_op,
10721072
int il,
1073-
ggml_tensor * probs_in) const {
1073+
ggml_tensor * probs_in,
1074+
ggml_tensor * gate_up_exps) const {
10741075
return build_moe_ffn(
10751076
cur,
10761077
gate_inp, /* gate_inp_b */ nullptr,
@@ -1086,7 +1087,8 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
10861087
w_scale,
10871088
gating_op,
10881089
il,
1089-
probs_in
1090+
probs_in,
1091+
gate_up_exps
10901092
);
10911093
}
10921094

@@ -1109,7 +1111,9 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
11091111
float w_scale,
11101112
llama_expert_gating_func_type gating_op,
11111113
int il,
1112-
ggml_tensor * probs_in) const {
1114+
ggml_tensor * probs_in,
1115+
ggml_tensor * gate_up_exps,
1116+
ggml_tensor * gate_up_exps_b) const {
11131117
const int64_t n_embd = cur->ne[0];
11141118
const int64_t n_tokens = cur->ne[1];
11151119
const bool weight_before_ffn = arch == LLM_ARCH_LLAMA4; // for llama4, we apply the sigmoid-ed weights before the FFN
@@ -1248,38 +1252,60 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
12481252
cb(cur, "ffn_moe_weighted", il);
12491253
}
12501254

1251-
ggml_tensor * up = build_lora_mm_id(up_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens]
1252-
cb(up, "ffn_moe_up", il);
1255+
ggml_tensor * up = nullptr;
1256+
ggml_tensor * experts = nullptr;
12531257

1254-
if (up_exps_b) {
1255-
up = ggml_add_id(ctx0, up, up_exps_b, selected_experts);
1256-
cb(up, "ffn_moe_up_biased", il);
1257-
}
1258+
if (gate_up_exps) {
1259+
// merged gate_up path: one mul_mat_id, then split into gate and up views
1260+
ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, selected_experts); // [n_ff*2, n_expert_used, n_tokens]
1261+
cb(gate_up, "ffn_moe_gate_up", il);
12581262

1259-
ggml_tensor * experts = nullptr;
1260-
if (gate_exps) {
1261-
cur = build_lora_mm_id(gate_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens]
1263+
if (gate_up_exps_b) {
1264+
gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, selected_experts);
1265+
cb(gate_up, "ffn_moe_gate_up_biased", il);
1266+
}
1267+
1268+
const int64_t n_ff = gate_up->ne[0] / 2;
1269+
cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], 0);
12621270
cb(cur, "ffn_moe_gate", il);
1271+
up = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], n_ff * gate_up->nb[0]);
1272+
cb(up, "ffn_moe_up", il);
12631273
} else {
1264-
cur = up;
1265-
}
1274+
// separate gate and up path
1275+
up = build_lora_mm_id(up_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens]
1276+
cb(up, "ffn_moe_up", il);
12661277

1267-
if (gate_exps_b) {
1268-
cur = ggml_add_id(ctx0, cur, gate_exps_b, selected_experts);
1269-
cb(cur, "ffn_moe_gate_biased", il);
1278+
if (up_exps_b) {
1279+
up = ggml_add_id(ctx0, up, up_exps_b, selected_experts);
1280+
cb(up, "ffn_moe_up_biased", il);
1281+
}
1282+
1283+
if (gate_exps) {
1284+
cur = build_lora_mm_id(gate_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens]
1285+
cb(cur, "ffn_moe_gate", il);
1286+
} else {
1287+
cur = up;
1288+
}
1289+
1290+
if (gate_exps_b) {
1291+
cur = ggml_add_id(ctx0, cur, gate_exps_b, selected_experts);
1292+
cb(cur, "ffn_moe_gate_biased", il);
1293+
}
12701294
}
12711295

1296+
const bool has_gate = gate_exps || gate_up_exps;
1297+
12721298
switch (type_op) {
12731299
case LLM_FFN_SILU:
1274-
if (gate_exps) {
1300+
if (has_gate) {
12751301
cur = ggml_swiglu_split(ctx0, cur, up);
12761302
cb(cur, "ffn_moe_swiglu", il);
12771303
} else {
12781304
cur = ggml_silu(ctx0, cur);
12791305
cb(cur, "ffn_moe_silu", il);
12801306
} break;
12811307
case LLM_FFN_GELU:
1282-
if (gate_exps) {
1308+
if (has_gate) {
12831309
cur = ggml_geglu_split(ctx0, cur, up);
12841310
cb(cur, "ffn_moe_geglu", il);
12851311
} else {
@@ -1295,15 +1321,15 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
12951321
cb(cur, "ffn_moe_swiglu_oai", il);
12961322
} break;
12971323
case LLM_FFN_RELU:
1298-
if (gate_exps) {
1324+
if (has_gate) {
12991325
cur = ggml_reglu_split(ctx0, cur, up);
13001326
cb(cur, "ffn_moe_reglu", il);
13011327
} else {
13021328
cur = ggml_relu(ctx0, cur);
13031329
cb(cur, "ffn_moe_relu", il);
13041330
} break;
13051331
case LLM_FFN_RELU_SQR:
1306-
if (gate_exps) {
1332+
if (has_gate) {
13071333
// TODO: add support for gated squared relu
13081334
GGML_ABORT("fatal error: gated squared relu not implemented");
13091335
} else {

‎src/llama-graph.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -786,7 +786,8 @@ struct llm_graph_context {
786786
float w_scale,
787787
llama_expert_gating_func_type gating_op,
788788
int il,
789-
ggml_tensor * probs_in = nullptr) const;
789+
ggml_tensor * probs_in = nullptr,
790+
ggml_tensor * gate_up_exps = nullptr) const;
790791

791792
ggml_tensor * build_moe_ffn(
792793
ggml_tensor * cur,
@@ -807,7 +808,9 @@ struct llm_graph_context {
807808
float w_scale,
808809
llama_expert_gating_func_type gating_op,
809810
int il,
810-
ggml_tensor * probs_in = nullptr) const;
811+
ggml_tensor * probs_in = nullptr,
812+
ggml_tensor * gate_up_exps = nullptr,
813+
ggml_tensor * gate_up_exps_b = nullptr) const;
811814

812815
//
813816
// inputs

‎src/llama-model.cpp‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4983,9 +4983,14 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
49834983
}
49844984

49854985
// MoE branch
4986-
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);
49874986
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
4988-
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);
4987+
4988+
// try merged gate_up first, fall back to separate gate and up
4989+
layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, n_ff_exp * 2, n_expert}, TENSOR_NOT_REQUIRED);
4990+
if (layer.ffn_gate_up_exps == nullptr) {
4991+
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
4992+
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
4993+
}
49894994

49904995
// Shared expert branch
49914996
layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
@@ -6527,9 +6532,14 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
65276532
layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0);
65286533

65296534
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert}, 0);
6530-
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);
65316535
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
6532-
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0);
6536+
6537+
// try merged gate_up first, fall back to separate gate and up
6538+
layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, n_ff_exp * 2, n_expert}, TENSOR_NOT_REQUIRED);
6539+
if (layer.ffn_gate_up_exps == nullptr) {
6540+
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
6541+
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
6542+
}
65336543

65346544
// bias
65356545
layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_head * n_rot}, 0);
@@ -6538,9 +6548,14 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
65386548
layer.bo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
65396549

65406550
layer.ffn_gate_inp_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "bias", i), {n_expert}, 0);
6541-
layer.ffn_gate_exps_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "bias", i), {n_ff_exp, n_expert}, 0);
65426551
layer.ffn_down_exps_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "bias", i), { n_embd, n_expert}, 0);
6543-
layer.ffn_up_exps_b = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "bias", i), {n_ff_exp, n_expert}, 0);
6552+
6553+
// try merged gate_up bias first, fall back to separate gate and up
6554+
layer.ffn_gate_up_exps_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "bias", i), {n_ff_exp * 2, n_expert}, TENSOR_NOT_REQUIRED);
6555+
if (layer.ffn_gate_up_exps_b == nullptr) {
6556+
layer.ffn_gate_exps_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "bias", i), {n_ff_exp, n_expert}, 0);
6557+
layer.ffn_up_exps_b = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "bias", i), {n_ff_exp, n_expert}, 0);
6558+
}
65446559
}
65456560
} break;
65466561
case LLM_ARCH_LFM2:

‎src/llama-model.h‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -275,14 +275,16 @@ struct llama_layer {
275275
struct ggml_tensor * ffn_up_enc = nullptr;
276276

277277
// ff MoE
278-
struct ggml_tensor * ffn_gate_inp = nullptr;
279-
struct ggml_tensor * ffn_gate_exps = nullptr;
280-
struct ggml_tensor * ffn_down_exps = nullptr;
281-
struct ggml_tensor * ffn_up_exps = nullptr;
282-
struct ggml_tensor * ffn_gate_inp_b = nullptr;
283-
struct ggml_tensor * ffn_gate_exps_b = nullptr;
284-
struct ggml_tensor * ffn_down_exps_b = nullptr;
285-
struct ggml_tensor * ffn_up_exps_b = nullptr;
278+
struct ggml_tensor * ffn_gate_inp = nullptr;
279+
struct ggml_tensor * ffn_gate_exps = nullptr;
280+
struct ggml_tensor * ffn_down_exps = nullptr;
281+
struct ggml_tensor * ffn_up_exps = nullptr;
282+
struct ggml_tensor * ffn_gate_up_exps = nullptr;
283+
struct ggml_tensor * ffn_gate_inp_b = nullptr;
284+
struct ggml_tensor * ffn_gate_exps_b = nullptr;
285+
struct ggml_tensor * ffn_down_exps_b = nullptr;
286+
struct ggml_tensor * ffn_up_exps_b = nullptr;
287+
struct ggml_tensor * ffn_gate_up_exps_b = nullptr;
286288

287289
// ff shared expert (shexp)
288290
struct ggml_tensor * ffn_gate_inp_shexp = nullptr;

0 commit comments

Comments
 (0)