-
Notifications
You must be signed in to change notification settings - Fork 236
Expand file tree
/
Copy pathmodels_atom.yaml
More file actions
40 lines (37 loc) · 2.17 KB
/
Copy pathmodels_atom.yaml
File metadata and controls
40 lines (37 loc) · 2.17 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
# Model-specific ATOM server configurations for disaggregated inference.
#
# Each top-level key is a MODEL_NAME value (must match the directory name under MODEL_DIR).
#
# To add a new model: add a new top-level entry following the same schema.
# No script changes are required.
#
# Schema:
# <model-name>:
# env: str # Space-separated KEY=VALUE pairs exported unconditionally
# hf_overrides: str # JSON string passed to --hf-overrides
# tp_dp_flags: str # Parallel flags for TP+DPA case (must include --enable-dp-attention)
# tp_dp_env: str # Space-separated KEY=VALUE pairs exported only in TP+DPA mode
# ep_dp_flags: str # Parallel flags for EP+DPA case (must include --enable-expert-parallel --enable-dp-attention)
# ep_dp_env: str # Space-separated KEY=VALUE pairs exported only in EP+DPA mode
# mtp_flags: str # Flags passed to SPEC_ARGS before $DECODE_MTP_SIZE (e.g. "--method mtp --num-speculative-tokens")
# kv_cache_flags: str # Full --kv_cache_dtype flag string (e.g. "--kv_cache_dtype fp8", or "" for none)
DeepSeek-V4-Pro:
env: "ATOM_MOE_GU_ITLV=1 AITER_BF16_FP8_MOE_BOUND=0"
kv_cache_flags: "--kv_cache_dtype fp8"
tp_dp_flags: "--enable-dp-attention --enable-tbo"
tp_dp_env: "GPU_MAX_HW_QUEUES=5 ATOM_CPU_AFFINITY=1"
ep_dp_flags: "--enable-expert-parallel --enable-dp-attention"
mtp_flags: "--method mtp --num-speculative-tokens"
hf_overrides: '{"use_index_cache":true,"index_topk_freq":4}'
MiniMax-M3-MXFP4:
env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 ATOM_M3_SPARSE_USE_ASM_PA=1 AITER_QUICK_REDUCE_CAST_BF16_TO_FP16=0"
kv_cache_flags: "--kv_cache_dtype fp8"
tp_dp_flags: "--enable-dp-attention"
ep_dp_flags: "--enable-expert-parallel --enable-dp-attention"
mtp_flags: "--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens"
MiniMax-M3-MXFP8:
env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 ATOM_M3_SPARSE_USE_ASM_PA=1 AITER_QUICK_REDUCE_CAST_BF16_TO_FP16=0"
kv_cache_flags: "--kv_cache_dtype fp8"
tp_dp_flags: "--enable-dp-attention"
ep_dp_flags: "--enable-expert-parallel --enable-dp-attention"
mtp_flags: "--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens"