Skip to content

Commit 8c146a8

Browse files
am17anpwilkinsszymczyngxsonfairydreaming
authored
DeepSeek V4 (ggml-org#24162)
* convert: add dsv4 conversion * add basic setup * add llm_graph_input_dsv4 * add save-load state * add sinkhorn eps - correction by @fairydreaming * add rope fix * cleanup dead code * fix bugs * support pro model: added by @fairydreaming * remove redundant V cache * Chat template * remove debugging leftovers * Add mechanism for inlining templates based on architecture * s/deepseek-v4-flash/deepseek4/g * s/deepseek-v4-flash/deepseek4/g continued * enable graph reuse * enable FA * fix test llama archs * rename * compatibility with antirez ds4 GGUFs * simplified set_gguf_parameters() by calling super class method, replaced moe.score_func with expert_gating_func. * reserve worst-case kv-cache * revert max split inputs * address review comments * add padding to enable FA * pad only the final value of plan.n_kv to 256 * remove built-in cpp chat template * cont: remove cpp built-in template * rm outdated test * replace ggml_view_3d() with ggml_reshape_3d() Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> * only support n_seq=1 for now * remove unused var * cont: remove unused var * use scale bias * use correct ptr for can_reuse * remove gen-chat-inline-templates.py * simplify graph reuse * cont: cleanup * remove unused inputs * enable partial checkpointing * add correct shape for kq_mask + set llama_model_n_swa to 0 for dsv4 * precompute source_idx + add comment about dummy write * support multi-seq * remove restored_trim_pos * use split_equal when possible * fix indent * address review comments * use LLM_KV * fix ci --------- Co-authored-by: Piotr Wilkin <piotr.wilkin@syndatis.com> Co-authored-by: Stanisław Szymczyk <sszymczy@gmail.com> Co-authored-by: Xuan Son Nguyen <son@huggingface.co> Co-authored-by: fairydreaming <166155368+fairydreaming@users.noreply.github.com> Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
1 parent 6cb18b2 commit 8c146a8

25 files changed

Lines changed: 4698 additions & 40 deletions

‎conversion/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
"DeepseekV3ForCausalLM": "deepseek",
5252
"DeepseekV32ForCausalLM": "deepseek",
5353
"DFlashDraftModel": "qwen",
54+
"DeepseekV4ForCausalLM": "deepseek",
5455
"DistilBertForMaskedLM": "bert",
5556
"DistilBertForSequenceClassification": "bert",
5657
"DistilBertModel": "bert",

‎conversion/base.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1273,7 +1273,7 @@ def set_gguf_parameters(self):
12731273
if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], optional=True)) is not None:
12741274
self.gguf_writer.add_layer_norm_eps(f_norm_eps)
12751275
logger.info(f"gguf: layer norm epsilon = {f_norm_eps}")
1276-
if (n_experts := self.find_hparam(["num_local_experts", "num_experts"], optional=True)) is not None:
1276+
if (n_experts := self.find_hparam(["num_local_experts", "num_experts", "n_routed_experts"], optional=True)) is not None:
12771277
self.gguf_writer.add_expert_count(n_experts)
12781278
logger.info(f"gguf: expert count = {n_experts}")
12791279
if (n_experts_used := self.find_hparam(["num_experts_per_tok", "num_experts_per_token", "top_k_experts"], optional=True)) is not None:
@@ -1291,6 +1291,8 @@ def set_gguf_parameters(self):
12911291
self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
12921292
elif score_func == "softmax":
12931293
self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX)
1294+
elif score_func == "sqrtsoftplus":
1295+
self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SQRTSOFTPLUS)
12941296
else:
12951297
raise ValueError(f"Unsupported expert score gating function value: {score_func}")
12961298
logger.info(f"gguf: expert score gating function = {score_func}")
@@ -2600,6 +2602,17 @@ def __torch_function__(cls, func, types, args=(), kwargs=None):
26002602
return cls._wrap_fn(func)(*args, **kwargs)
26012603

26022604

2605+
if hasattr(torch, "float8_e8m0fnu"):
2606+
_torch_float8_e8m0 = torch.float8_e8m0fnu
2607+
LazyTorchTensor._dtype_map[_torch_float8_e8m0] = np.uint8
2608+
LazyTorchTensor._dtype_byteswap_map[_torch_float8_e8m0] = np.uint8
2609+
LazyTorchTensor._dtype_str_map["F8_E8M0"] = _torch_float8_e8m0
2610+
else:
2611+
# Older torch builds do not expose F8_E8M0. Keep the raw bytes so callers
2612+
# that know the format can decode them explicitly.
2613+
LazyTorchTensor._dtype_str_map["F8_E8M0"] = torch.uint8
2614+
2615+
26032616
def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> str:
26042617
# TODO @ngxson : this won't work correctly if the model has both audio & vision encoders
26052618
# maybe we should fallback to text model's arch in that case, since not many models have both

‎conversion/deepseek.py‎

Lines changed: 308 additions & 1 deletion
Large diffs are not rendered by default.

‎gguf-py/gguf/constants.py‎

Lines changed: 102 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ class LLM:
145145
TOKEN_SHIFT_COUNT = "{arch}.token_shift_count"
146146
INTERLEAVE_MOE_LAYER_STEP = "{arch}.interleave_moe_layer_step"
147147
FULL_ATTENTION_INTERVAL = "{arch}.full_attention_interval"
148+
HASH_LAYER_COUNT = "{arch}.hash_layer_count"
148149
ACTIVATION_SPARSITY_SCALE = "{arch}.activation_sparsity_scale"
149150
ALTUP_ACTIVE_IDX = "{arch}.altup.active_idx"
150151
ALTUP_NUM_INPUTS = "{arch}.altup.num_inputs"
@@ -180,8 +181,12 @@ class Attention:
180181
REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count"
181182
SLIDING_WINDOW = "{arch}.attention.sliding_window"
182183
SCALE = "{arch}.attention.scale"
184+
OUTPUT_GROUP_COUNT = "{arch}.attention.output_group_count"
185+
OUTPUT_LORA_RANK = "{arch}.attention.output_lora_rank"
183186
OUTPUT_SCALE = "{arch}.attention.output_scale"
184187
VALUE_SCALE = "{arch}.attention.value_scale"
188+
COMPRESS_RATIOS = "{arch}.attention.compress_ratios"
189+
COMPRESS_ROPE_FREQ_BASE = "{arch}.attention.compress_rope_freq_base"
185190
TEMPERATURE_LENGTH = "{arch}.attention.temperature_length"
186191
KEY_LENGTH_MLA = "{arch}.attention.key_length_mla"
187192
VALUE_LENGTH_MLA = "{arch}.attention.value_length_mla"
@@ -196,6 +201,11 @@ class Indexer:
196201
KEY_LENGTH = "{arch}.attention.indexer.key_length"
197202
TOP_K = "{arch}.attention.indexer.top_k"
198203

204+
class HyperConnection:
205+
COUNT = "{arch}.hyper_connection.count"
206+
SINKHORN_ITERATIONS = "{arch}.hyper_connection.sinkhorn_iterations"
207+
EPSILON = "{arch}.hyper_connection.epsilon"
208+
199209
class Rope:
200210
DIMENSION_COUNT = "{arch}.rope.dimension_count"
201211
DIMENSION_COUNT_SWA = "{arch}.rope.dimension_count_swa"
@@ -470,6 +480,7 @@ class MODEL_ARCH(IntEnum):
470480
DEEPSEEK2 = auto()
471481
DEEPSEEK2OCR = auto()
472482
DEEPSEEK32 = auto()
483+
DEEPSEEK4 = auto()
473484
CHATGLM = auto()
474485
GLM4 = auto()
475486
GLM4_MOE = auto()
@@ -555,6 +566,9 @@ class MODEL_TENSOR(IntEnum):
555566
DENSE_2_OUT = auto() # embeddinggemma 2_Dense
556567
DENSE_3_OUT = auto() # embeddinggemma 3_Dense
557568
OUTPUT_NORM = auto()
569+
HC_HEAD_FN = auto()
570+
HC_HEAD_BASE = auto()
571+
HC_HEAD_SCALE = auto()
558572
ROPE_FREQS = auto()
559573
ROPE_FACTORS_LONG = auto()
560574
ROPE_FACTORS_SHORT = auto()
@@ -594,6 +608,7 @@ class MODEL_TENSOR(IntEnum):
594608
FFN_DOWN_CHEXP = auto()
595609
FFN_UP_CHEXP = auto()
596610
FFN_EXP_PROBS_B = auto()
611+
FFN_GATE_TID2EID = auto()
597612
MOE_LATENT_DOWN = auto() # nemotron 3 super
598613
MOE_LATENT_UP = auto() # nemotron 3 super
599614
ATTN_Q_NORM = auto()
@@ -681,6 +696,20 @@ class MODEL_TENSOR(IntEnum):
681696
ATTN_V_B = auto()
682697
ATTN_Q_A_NORM = auto()
683698
ATTN_KV_A_NORM = auto()
699+
ATTN_KV = auto()
700+
ATTN_KV_NORM = auto()
701+
ATTN_OUT_A = auto()
702+
ATTN_OUT_B = auto()
703+
HC_ATTN_FN = auto()
704+
HC_ATTN_BASE = auto()
705+
HC_ATTN_SCALE = auto()
706+
HC_FFN_FN = auto()
707+
HC_FFN_BASE = auto()
708+
HC_FFN_SCALE = auto()
709+
ATTN_COMPRESSOR_WKV = auto()
710+
ATTN_COMPRESSOR_WGATE = auto()
711+
ATTN_COMPRESSOR_APE = auto()
712+
ATTN_COMPRESSOR_NORM = auto()
684713
FFN_SUB_NORM = auto()
685714
ATTN_SUB_NORM = auto()
686715
DEC_ATTN_NORM = auto()
@@ -742,6 +771,10 @@ class MODEL_TENSOR(IntEnum):
742771
INDEXER_PROJ = auto()
743772
INDEXER_ATTN_K = auto()
744773
INDEXER_ATTN_Q_B = auto()
774+
INDEXER_COMPRESSOR_WKV = auto()
775+
INDEXER_COMPRESSOR_WGATE = auto()
776+
INDEXER_COMPRESSOR_APE = auto()
777+
INDEXER_COMPRESSOR_NORM = auto()
745778
# vision
746779
V_MMPROJ = auto()
747780
V_MMPROJ_FC = auto()
@@ -1027,6 +1060,7 @@ class MODEL_TENSOR(IntEnum):
10271060
MODEL_ARCH.DEEPSEEK2: "deepseek2",
10281061
MODEL_ARCH.DEEPSEEK2OCR: "deepseek2-ocr",
10291062
MODEL_ARCH.DEEPSEEK32: "deepseek32",
1063+
MODEL_ARCH.DEEPSEEK4: "deepseek4",
10301064
MODEL_ARCH.CHATGLM: "chatglm",
10311065
MODEL_ARCH.GLM4: "glm4",
10321066
MODEL_ARCH.GLM4_MOE: "glm4moe",
@@ -1111,6 +1145,9 @@ class MODEL_TENSOR(IntEnum):
11111145
MODEL_TENSOR.OUTPUT: "output",
11121146
MODEL_TENSOR.DENSE_2_OUT: "dense_2", # embeddinggemma 2_Dense
11131147
MODEL_TENSOR.DENSE_3_OUT: "dense_3", # embeddinggemma 2_Dense
1148+
MODEL_TENSOR.HC_HEAD_FN: "output_hc_fn",
1149+
MODEL_TENSOR.HC_HEAD_BASE: "output_hc_base",
1150+
MODEL_TENSOR.HC_HEAD_SCALE: "output_hc_scale",
11141151
MODEL_TENSOR.ROPE_FREQS: "rope_freqs",
11151152
MODEL_TENSOR.ROPE_FACTORS_LONG: "rope_factors_long",
11161153
MODEL_TENSOR.ROPE_FACTORS_SHORT: "rope_factors_short",
@@ -1152,6 +1189,7 @@ class MODEL_TENSOR(IntEnum):
11521189
MODEL_TENSOR.FFN_UP_EXP: "blk.{bid}.ffn_up_exps",
11531190
MODEL_TENSOR.FFN_GATE_UP_EXP: "blk.{bid}.ffn_gate_up_exps",
11541191
MODEL_TENSOR.FFN_EXP_PROBS_B: "blk.{bid}.exp_probs_b",
1192+
MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid",
11551193
MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 super
11561194
MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super
11571195
MODEL_TENSOR.LAYER_OUT_NORM: "blk.{bid}.layer_output_norm",
@@ -1237,6 +1275,20 @@ class MODEL_TENSOR(IntEnum):
12371275
MODEL_TENSOR.ATTN_V_B: "blk.{bid}.attn_v_b",
12381276
MODEL_TENSOR.ATTN_Q_A_NORM: "blk.{bid}.attn_q_a_norm",
12391277
MODEL_TENSOR.ATTN_KV_A_NORM: "blk.{bid}.attn_kv_a_norm",
1278+
MODEL_TENSOR.ATTN_KV: "blk.{bid}.attn_kv",
1279+
MODEL_TENSOR.ATTN_KV_NORM: "blk.{bid}.attn_kv_a_norm",
1280+
MODEL_TENSOR.ATTN_OUT_A: "blk.{bid}.attn_output_a",
1281+
MODEL_TENSOR.ATTN_OUT_B: "blk.{bid}.attn_output_b",
1282+
MODEL_TENSOR.HC_ATTN_FN: "blk.{bid}.hc_attn_fn",
1283+
MODEL_TENSOR.HC_ATTN_BASE: "blk.{bid}.hc_attn_base",
1284+
MODEL_TENSOR.HC_ATTN_SCALE: "blk.{bid}.hc_attn_scale",
1285+
MODEL_TENSOR.HC_FFN_FN: "blk.{bid}.hc_ffn_fn",
1286+
MODEL_TENSOR.HC_FFN_BASE: "blk.{bid}.hc_ffn_base",
1287+
MODEL_TENSOR.HC_FFN_SCALE: "blk.{bid}.hc_ffn_scale",
1288+
MODEL_TENSOR.ATTN_COMPRESSOR_WKV: "blk.{bid}.attn_compressor_kv",
1289+
MODEL_TENSOR.ATTN_COMPRESSOR_WGATE: "blk.{bid}.attn_compressor_gate",
1290+
MODEL_TENSOR.ATTN_COMPRESSOR_APE: "blk.{bid}.attn_compressor_ape",
1291+
MODEL_TENSOR.ATTN_COMPRESSOR_NORM: "blk.{bid}.attn_compressor_norm",
12401292
MODEL_TENSOR.ATTN_SUB_NORM: "blk.{bid}.attn_sub_norm",
12411293
MODEL_TENSOR.FFN_SUB_NORM: "blk.{bid}.ffn_sub_norm",
12421294
MODEL_TENSOR.DEC_ATTN_NORM: "dec.blk.{bid}.attn_norm",
@@ -1298,6 +1350,10 @@ class MODEL_TENSOR(IntEnum):
12981350
MODEL_TENSOR.INDEXER_PROJ: "blk.{bid}.indexer.proj",
12991351
MODEL_TENSOR.INDEXER_ATTN_K: "blk.{bid}.indexer.attn_k",
13001352
MODEL_TENSOR.INDEXER_ATTN_Q_B: "blk.{bid}.indexer.attn_q_b",
1353+
MODEL_TENSOR.INDEXER_COMPRESSOR_WKV: "blk.{bid}.indexer_compressor_kv",
1354+
MODEL_TENSOR.INDEXER_COMPRESSOR_WGATE: "blk.{bid}.indexer_compressor_gate",
1355+
MODEL_TENSOR.INDEXER_COMPRESSOR_APE: "blk.{bid}.indexer_compressor_ape",
1356+
MODEL_TENSOR.INDEXER_COMPRESSOR_NORM: "blk.{bid}.indexer_compressor_norm",
13011357
# vision
13021358
MODEL_TENSOR.V_MMPROJ: "mm.{bid}",
13031359
MODEL_TENSOR.V_MMPROJ_FC: "mm.model.fc",
@@ -3138,6 +3194,49 @@ class MODEL_TENSOR(IntEnum):
31383194
MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
31393195
MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
31403196
],
3197+
MODEL_ARCH.DEEPSEEK4: [
3198+
MODEL_TENSOR.TOKEN_EMBD,
3199+
MODEL_TENSOR.OUTPUT_NORM,
3200+
MODEL_TENSOR.OUTPUT,
3201+
MODEL_TENSOR.HC_HEAD_FN,
3202+
MODEL_TENSOR.HC_HEAD_BASE,
3203+
MODEL_TENSOR.HC_HEAD_SCALE,
3204+
MODEL_TENSOR.ATTN_NORM,
3205+
MODEL_TENSOR.ATTN_SINKS,
3206+
MODEL_TENSOR.ATTN_Q_A,
3207+
MODEL_TENSOR.ATTN_Q_B,
3208+
MODEL_TENSOR.ATTN_Q_A_NORM,
3209+
MODEL_TENSOR.ATTN_KV,
3210+
MODEL_TENSOR.ATTN_KV_NORM,
3211+
MODEL_TENSOR.ATTN_OUT_A,
3212+
MODEL_TENSOR.ATTN_OUT_B,
3213+
MODEL_TENSOR.HC_ATTN_FN,
3214+
MODEL_TENSOR.HC_ATTN_BASE,
3215+
MODEL_TENSOR.HC_ATTN_SCALE,
3216+
MODEL_TENSOR.HC_FFN_FN,
3217+
MODEL_TENSOR.HC_FFN_BASE,
3218+
MODEL_TENSOR.HC_FFN_SCALE,
3219+
MODEL_TENSOR.ATTN_COMPRESSOR_WKV,
3220+
MODEL_TENSOR.ATTN_COMPRESSOR_WGATE,
3221+
MODEL_TENSOR.ATTN_COMPRESSOR_APE,
3222+
MODEL_TENSOR.ATTN_COMPRESSOR_NORM,
3223+
MODEL_TENSOR.INDEXER_PROJ,
3224+
MODEL_TENSOR.INDEXER_ATTN_Q_B,
3225+
MODEL_TENSOR.INDEXER_COMPRESSOR_WKV,
3226+
MODEL_TENSOR.INDEXER_COMPRESSOR_WGATE,
3227+
MODEL_TENSOR.INDEXER_COMPRESSOR_APE,
3228+
MODEL_TENSOR.INDEXER_COMPRESSOR_NORM,
3229+
MODEL_TENSOR.FFN_GATE_INP,
3230+
MODEL_TENSOR.FFN_GATE_TID2EID,
3231+
MODEL_TENSOR.FFN_EXP_PROBS_B,
3232+
MODEL_TENSOR.FFN_NORM,
3233+
MODEL_TENSOR.FFN_GATE_EXP,
3234+
MODEL_TENSOR.FFN_DOWN_EXP,
3235+
MODEL_TENSOR.FFN_UP_EXP,
3236+
MODEL_TENSOR.FFN_GATE_SHEXP,
3237+
MODEL_TENSOR.FFN_DOWN_SHEXP,
3238+
MODEL_TENSOR.FFN_UP_SHEXP,
3239+
],
31413240
MODEL_ARCH.ERNIE4_5_MOE: [
31423241
MODEL_TENSOR.TOKEN_EMBD,
31433242
MODEL_TENSOR.OUTPUT_NORM,
@@ -4437,8 +4536,9 @@ class GGMLQuantizationType(IntEnum):
44374536

44384537

44394538
class ExpertGatingFuncType(IntEnum):
4440-
SOFTMAX = 1
4441-
SIGMOID = 2
4539+
SOFTMAX = 1
4540+
SIGMOID = 2
4541+
SQRTSOFTPLUS = 4
44424542

44434543

44444544
# TODO: add GGMLFileType from ggml_ftype in ggml.h

‎gguf-py/gguf/gguf_writer.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -715,6 +715,9 @@ def add_leading_dense_block_count(self, length: int) -> None:
715715
def add_full_attention_interval(self, interval: int) -> None:
716716
self.add_uint32(Keys.LLM.FULL_ATTENTION_INTERVAL.format(arch=self.arch), interval)
717717

718+
def add_hash_layer_count(self, count: int) -> None:
719+
self.add_uint32(Keys.LLM.HASH_LAYER_COUNT.format(arch=self.arch), count)
720+
718721
def add_feed_forward_length(self, length: int | Sequence[int]) -> None:
719722
if isinstance(length, int):
720723
self.add_uint32(Keys.LLM.FEED_FORWARD_LENGTH.format(arch=self.arch), length)
@@ -952,6 +955,27 @@ def add_target_hidden_size(self, value: int) -> None:
952955
def add_norm_before_residual(self, value: bool) -> None:
953956
self.add_bool(Keys.LLM.NORM_BEFORE_RESIDUAL.format(arch=self.arch), value)
954957

958+
def add_attention_output_group_count(self, count: int) -> None:
959+
self.add_uint32(Keys.Attention.OUTPUT_GROUP_COUNT.format(arch=self.arch), count)
960+
961+
def add_attention_output_lora_rank(self, length: int) -> None:
962+
self.add_uint32(Keys.Attention.OUTPUT_LORA_RANK.format(arch=self.arch), length)
963+
964+
def add_attention_compress_ratios(self, values: Sequence[int]) -> None:
965+
self.add_array(Keys.Attention.COMPRESS_RATIOS.format(arch=self.arch), values)
966+
967+
def add_attention_compress_rope_freq_base(self, value: float) -> None:
968+
self.add_float32(Keys.Attention.COMPRESS_ROPE_FREQ_BASE.format(arch=self.arch), value)
969+
970+
def add_hyper_connection_count(self, count: int) -> None:
971+
self.add_uint32(Keys.HyperConnection.COUNT.format(arch=self.arch), count)
972+
973+
def add_hyper_connection_sinkhorn_iterations(self, count: int) -> None:
974+
self.add_uint32(Keys.HyperConnection.SINKHORN_ITERATIONS.format(arch=self.arch), count)
975+
976+
def add_hyper_connection_epsilon(self, value: float) -> None:
977+
self.add_float32(Keys.HyperConnection.EPSILON.format(arch=self.arch), value)
978+
955979
def add_attention_scale(self, value: float) -> None:
956980
self.add_float32(Keys.Attention.SCALE.format(arch=self.arch), value)
957981

0 commit comments

Comments
 (0)