Skip to content

Commit 0c5fc6b

Browse files
committed
Support GLM5.2 and use GLM5.x related flags
Signed-off-by: Hollow Man <hollowman@opensuse.org>
1 parent a9eabd3 commit 0c5fc6b

1 file changed

Lines changed: 5 additions & 0 deletions

File tree

src/megatron/bridge/models/glm_moe_dsa/glm5_bridge.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,11 @@ def provider_bridge(self, hf_pretrained: PreTrainedCausalLM) -> MLAModelProvider
111111
provider.dsa_indexer_head_dim = hf_config.index_head_dim
112112
provider.dsa_indexer_n_heads = hf_config.index_n_heads
113113
provider.dsa_indexer_topk = hf_config.index_topk
114+
provider.dsa_indexer_rope_interleaved = True
115+
provider.dsa_indexer_rotate_activation = False
116+
provider.dsa_indexer_k_norm_epsilon = 1e-6
117+
provider.dsa_indexer_topk_freq = getattr(hf_config, "index_topk_freq", 1)
118+
provider.dsa_indexer_skip_topk_offset = getattr(hf_config, "index_skip_topk_offset", 0)
114119
provider.dsa_indexer_loss_coeff = 0.001
115120
provider.dsa_indexer_use_sparse_loss = True
116121

0 commit comments

Comments
 (0)