We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 8324f55 commit bc16288Copy full SHA for bc16288
1 file changed
src/MaxText/utils/ckpt_scripts/convert_qwen3_moe.py
@@ -46,7 +46,25 @@
46
"head_dim": 128,
47
"num_experts": 128,
48
"moe_intermediate_size": 1536,
49
- }
+ },
50
+ "qwen3-30b-a3b": {
51
+ "num_hidden_layers": 48,
52
+ "num_attention_heads": 32,
53
+ "num_key_value_heads": 4,
54
+ "hidden_size": 2048,
55
+ "head_dim": 128,
56
+ "num_experts": 128,
57
+ "moe_intermediate_size": 768,
58
59
+ "qwen3-480b-a35b": {
60
+ "num_hidden_layers": 62,
61
+ "num_attention_heads": 96,
62
+ "num_key_value_heads": 8,
63
+ "hidden_size": 6144,
64
65
+ "num_experts": 160,
66
+ "moe_intermediate_size": 2560,
67
68
}
69
70
0 commit comments