-
Notifications
You must be signed in to change notification settings - Fork 234
Expand file tree
/
Copy pathmodels_atom.yaml
More file actions
75 lines (72 loc) · 4.72 KB
/
Copy pathmodels_atom.yaml
File metadata and controls
75 lines (72 loc) · 4.72 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
# Model-specific ATOM server configurations for disaggregated inference.
#
# Each top-level key is a MODEL_NAME value (must match the directory name under MODEL_DIR).
#
# To add a new model: add a new top-level entry following the same schema.
# No script changes are required.
#
# Schema:
# <model-name>:
# env: str # Space-separated KEY=VALUE pairs exported unconditionally
# tp_dp_flags: str # Shared TP+DPA flags (fallback when prefill/decode-specific keys are absent)
# prefill_tp_dp_flags: str # TP+DPA flags for prefill only (overrides tp_dp_flags)
# decode_tp_dp_flags: str # TP+DPA flags for decode only (overrides tp_dp_flags)
# tp_dp_env: str # Space-separated KEY=VALUE pairs exported only in TP+DPA mode
# ep_dp_flags: str # Shared EP+DPA flags (fallback when prefill/decode-specific keys are absent)
# prefill_ep_dp_flags: str # EP+DPA flags for prefill only (overrides ep_dp_flags)
# decode_ep_dp_flags: str # EP+DPA flags for decode only (overrides ep_dp_flags)
# ep_dp_env: str # Space-separated KEY=VALUE pairs exported only in EP+DPA mode
# mtp_flags: str # Flags passed to SPEC_ARGS before $DECODE_MTP_SIZE (e.g. "--method mtp --num-speculative-tokens")
# kv_cache_flags: str # Full --kv_cache_dtype flag string (e.g. "--kv_cache_dtype fp8", or "" for none)
# online_quant_config: str # JSON string passed to --online_quant_config (used when DPA is disabled)
# online_quant_dpa_config: str # JSON string passed to --online_quant_config when DPA is enabled (falls back to online_quant_config)
# block_size: str # --block-size value (overrides server_atom.sh default of 16)
# mem_frac_static: str # --gpu-memory-utilization value (overrides default of 0.85)
# max_model_len: str # --max-model-len value (overrides default of unset)
# max_num_seqs: str # --max-num-seqs value (overrides default of 256)
# max_num_batched_tokens: str # --max-num-batched-tokens value (overrides default of unset)
DeepSeek-V4-Pro:
env: "ATOM_MOE_GU_ITLV=1 AITER_BF16_FP8_MOE_BOUND=0"
kv_cache_flags: "--kv_cache_dtype fp8"
tp_dp_flags: "--enable-dp-attention --enable-tbo"
prefill_tp_dp_flags: "--enable-dp-attention --enable-tbo"
decode_tp_dp_flags: "--enable-dp-attention --enable-tbo"
tp_dp_env: "GPU_MAX_HW_QUEUES=5 ATOM_CPU_AFFINITY=1"
ep_dp_flags: "--enable-expert-parallel --enable-dp-attention"
prefill_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention"
decode_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention"
mtp_flags: "--method mtp --num-speculative-tokens"
MiniMax-M3-MXFP4:
env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 ATOM_FORCE_ATTN_TRITON=1"
kv_cache_flags: "--kv_cache_dtype fp8"
tp_dp_flags: "--enable-dp-attention"
prefill_tp_dp_flags: "--enable-dp-attention --enable-tbo prefill"
decode_tp_dp_flags: "--enable-dp-attention"
ep_dp_flags: "--enable-expert-parallel --enable-dp-attention"
prefill_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention"
decode_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention"
mtp_flags: "--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens"
online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}'
online_quant_dpa_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}'
block_size: "128"
mem_frac_static: "0.8"
max_model_len: "32768"
max_num_seqs: "256"
max_num_batched_tokens: "32768"
MiniMax-M3-MXFP8:
env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 ATOM_FORCE_ATTN_TRITON=1"
kv_cache_flags: "--kv_cache_dtype fp8"
tp_dp_flags: "--enable-dp-attention"
prefill_tp_dp_flags: "--enable-dp-attention --enable-tbo prefill"
decode_tp_dp_flags: "--enable-dp-attention"
ep_dp_flags: "--enable-expert-parallel --enable-dp-attention"
prefill_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention"
decode_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention"
mtp_flags: "--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens"
online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}'
online_quant_dpa_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*.gate.*","*.block_sparse_moe.experts*"]}'
block_size: "128"
mem_frac_static: "0.8"
max_model_len: "32768"
max_num_seqs: "256"
max_num_batched_tokens: "32768"