Skip to content

Commit 786f856

Browse files
cquil11cquil11
committed
Merge main into agent/bump-aiperf-main-deterministic-warmup
Preserve the PR 2415 performance changelog entry at the tail after resolving the append-only changelog conflict. 中文：合并 main，并在解决仅追加性能变更日志冲突后，将 PR 2415 条目保留在文件末尾。
2 parents 9361b5c + b6abf19 commit 786f856

37 files changed

Lines changed: 5322 additions & 92 deletions

File tree

‎benchmarks/benchmark_lib.sh‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1812,7 +1812,11 @@ build_replay_cmd() {
18121812
# aiperf's conv-aware routing emits nvext.session_control, a removed POC field
18131813
# (dynamo #9920 / v1.3.0-dev) that current dynamo builds reject with a 400
18141814
# (they moved to router/routing_constraints/agent_context). Default stays on.
1815-
if [[ "${FRAMEWORK:-}" == dynamo-* && "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" ]]; then
1815+
# New recipes instead set AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=true
1816+
# to route by X-Dynamo-Session-ID header, which needs no routing CLI flag.
1817+
if [[ "${FRAMEWORK:-}" == dynamo-* \
1818+
&& "${AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING:-1}" != "0" \
1819+
&& "${AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID:-false}" != "true" ]]; then
18161820
REPLAY_CMD+=" --use-dynamo-conv-aware-routing"
18171821
# The upstream 300s affinity TTL is shorter than an overloaded
18181822
# high-concurrency agentic request. Keep bindings alive across long

‎benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8-mtp.yaml‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
name: "disagg-b300-1p1d-dep4-dep8-mtp"
2+
3+
frontend:
4+
type: dynamo
5+
enable_multiple_frontends: true
6+
num_additional_frontends: 8
7+
8+
dynamo:
9+
hash: "41882ae9b07232eed4850fb1daf8c958abb2556a"
10+
install: true
11+
12+
model:
13+
path: "deepseek-v4-pro"
14+
container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4"
15+
precision: "fp4"
16+
17+
sbatch_directives:
18+
cpus-per-task: "144"
19+
mem: "0"
20+
21+
resources:
22+
gpu_type: "b300"
23+
gpus_per_node: 8
24+
prefill_nodes: 1
25+
prefill_workers: 1
26+
gpus_per_prefill: 4
27+
decode_nodes: 1
28+
decode_workers: 1
29+
gpus_per_decode: 8
30+
31+
backend:
32+
type: sglang
33+
34+
prefill_environment:
35+
PYTHONUNBUFFERED: "1"
36+
SGLANG_RADIX_FORCE_MISS: "1"
37+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
38+
SGLANG_DEFAULT_THINKING: "1"
39+
SGLANG_DSV4_REASONING_EFFORT: "max"
40+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
41+
SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1"
42+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
43+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
44+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216"
45+
NCCL_MNNVL_ENABLE: "1"
46+
NCCL_CUMEM_ENABLE: "1"
47+
MC_FORCE_MNNVL: "1"
48+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
49+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
50+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
51+
UCX_TLS: "cuda_copy,rc"
52+
53+
decode_environment:
54+
PYTHONUNBUFFERED: "1"
55+
SGLANG_RADIX_FORCE_MISS: "1"
56+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
57+
SGLANG_DEFAULT_THINKING: "1"
58+
SGLANG_DSV4_REASONING_EFFORT: "max"
59+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
60+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
61+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
62+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048"
63+
NCCL_MNNVL_ENABLE: "1"
64+
NCCL_CUMEM_ENABLE: "1"
65+
MC_FORCE_MNNVL: "1"
66+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
67+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
68+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
69+
UCX_TLS: "cuda_copy,rc"
70+
SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0"
71+
72+
sglang_config:
73+
prefill:
74+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
75+
model-path: "/model/"
76+
trust-remote-code: true
77+
tool-call-parser: deepseekv4
78+
79+
disaggregation-mode: "prefill"
80+
watchdog-timeout: 86400
81+
disaggregation-transfer-backend: nixl
82+
83+
tensor-parallel-size: 4
84+
data-parallel-size: 4
85+
expert-parallel-size: 4
86+
87+
enable-dp-attention: true
88+
enable-dp-lm-head: true
89+
moe-a2a-backend: "megamoe"
90+
mem-fraction-static: 0.9
91+
max-running-requests: 256
92+
cuda-graph-max-bs: 256
93+
chunked-prefill-size: 32768
94+
stream-interval: 60
95+
96+
decode:
97+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
98+
model-path: "/model/"
99+
trust-remote-code: true
100+
tool-call-parser: deepseekv4
101+
102+
disaggregation-mode: "decode"
103+
watchdog-timeout: 86400
104+
disaggregation-transfer-backend: nixl
105+
106+
tensor-parallel-size: 8
107+
data-parallel-size: 8
108+
expert-parallel-size: 8
109+
110+
enable-dp-attention: true
111+
enable-dp-lm-head: true
112+
moe-a2a-backend: "megamoe"
113+
114+
speculative-algo: "EAGLE"
115+
speculative-num-steps: 3
116+
speculative-eagle-topk: 1
117+
speculative-num-draft-tokens: 4
118+
119+
mem-fraction-static: 0.94
120+
max-running-requests: 3072
121+
cuda-graph-max-bs: 256
122+
swa-full-tokens-ratio: 0.15
123+
context-length: 16384
124+
stream-interval: 60
125+
126+
benchmark:
127+
type: "sa-bench"
128+
isl: 8192
129+
osl: 1024
130+
random_range_ratio: 0.8
131+
concurrencies: "256"
132+
req_rate: "inf"
133+
use_chat_template: true
134+
custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"

‎benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-b300-1p1d-dep4-dep8.yaml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
name: "disagg-b300-1p1d-dep4-dep8"
2+
3+
model:
4+
path: "deepseek-v4-pro"
5+
container: "lmsysorg/sglang:nightly-dev-cu13-20260721-8905cbd4"
6+
precision: "fp4"
7+
8+
dynamo:
9+
hash: "41882ae9b07232eed4850fb1daf8c958abb2556a"
10+
install: true
11+
12+
sbatch_directives:
13+
cpus-per-task: "144"
14+
mem: "0"
15+
16+
resources:
17+
gpu_type: "b300"
18+
gpus_per_node: 8
19+
prefill_nodes: 1
20+
prefill_workers: 1
21+
gpus_per_prefill: 4
22+
decode_nodes: 1
23+
decode_workers: 1
24+
gpus_per_decode: 8
25+
26+
frontend:
27+
type: dynamo
28+
enable_multiple_frontends: true
29+
num_additional_frontends: 8
30+
env:
31+
args:
32+
router-mode: "kv"
33+
router-kv-overlap-score-weight: 0
34+
router-queue-threshold: 64
35+
router-temperature: 0.5
36+
no-kv-events: true
37+
38+
backend:
39+
type: sglang
40+
41+
prefill_environment:
42+
PYTHONUNBUFFERED: "1"
43+
SGLANG_RADIX_FORCE_MISS: "1"
44+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
45+
SGLANG_DEFAULT_THINKING: "1"
46+
SGLANG_DSV4_REASONING_EFFORT: "max"
47+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
48+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192"
49+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
50+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
51+
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
52+
NCCL_MNNVL_ENABLE: "1"
53+
NCCL_CUMEM_ENABLE: "1"
54+
MC_FORCE_MNNVL: "1"
55+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
56+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
57+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
58+
UCX_TLS: "cuda_copy,rc"
59+
DYN_SKIP_SGLANG_LOG_FORMATTING: "1"
60+
SGLANG_LOG_FORWARD_ITERS: "1"
61+
SGLANG_LOG_MS: "1"
62+
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"
63+
64+
decode_environment:
65+
PYTHONUNBUFFERED: "1"
66+
SGLANG_RADIX_FORCE_MISS: "1"
67+
SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1"
68+
SGLANG_DEFAULT_THINKING: "1"
69+
SGLANG_DSV4_REASONING_EFFORT: "max"
70+
SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1"
71+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280"
72+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1"
73+
SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1"
74+
SGLANG_OPT_USE_ONLINE_COMPRESS: "1"
75+
NCCL_MNNVL_ENABLE: "1"
76+
NCCL_CUMEM_ENABLE: "1"
77+
SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8"
78+
MC_FORCE_MNNVL: "1"
79+
SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000"
80+
SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000"
81+
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1"
82+
UCX_TLS: "cuda_copy,rc"
83+
DYN_SKIP_SGLANG_LOG_FORMATTING: "1"
84+
SGLANG_LOG_FORWARD_ITERS: "1"
85+
SGLANG_LOG_MS: "1"
86+
SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60"
87+
88+
sglang_config:
89+
prefill:
90+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
91+
trust-remote-code: true
92+
watchdog-timeout: 86400
93+
skip-tokenizer-init: true
94+
stream-interval: 60
95+
96+
tensor-parallel-size: 4
97+
data-parallel-size: 4
98+
expert-parallel-size: 4
99+
100+
enable-dp-attention: true
101+
moe-a2a-backend: "megamoe"
102+
moe-dense-tp-size: 1
103+
104+
disaggregation-mode: "prefill"
105+
disaggregation-transfer-backend: nixl
106+
enable-dp-lm-head: true
107+
108+
mem-fraction-static: 0.90
109+
max-running-requests: 512
110+
cuda-graph-max-bs: 512
111+
chunked-prefill-size: 32768
112+
113+
decode:
114+
served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
115+
trust-remote-code: true
116+
watchdog-timeout: 86400
117+
skip-tokenizer-init: true
118+
stream-interval: 60
119+
120+
moe-a2a-backend: "megamoe"
121+
moe-dense-tp-size: 1
122+
123+
disaggregation-mode: "decode"
124+
disaggregation-transfer-backend: nixl
125+
disaggregation-decode-polling-interval: 8
126+
127+
mem-fraction-static: 0.94
128+
swa-full-tokens-ratio: 0.20
129+
context-length: 9216
130+
tensor-parallel-size: 8
131+
data-parallel-size: 8
132+
expert-parallel-size: 8
133+
enable-dp-attention: true
134+
enable-dp-lm-head: true
135+
max-running-requests: 18432
136+
cuda-graph-max-bs: 1280
137+
138+
benchmark:
139+
type: "sa-bench"
140+
isl: 8192
141+
osl: 1024
142+
concurrencies: "256"
143+
req_rate: "inf"
144+
use_chat_template: false
145+
custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer"

0 commit comments

Comments
 (0)