Skip to content

Commit a4bb43a

Browse files
richardhuo-nvadibarra
richardhuo-nv
and
adibarra
authored
[NV] Kimi K2.5 NVFP4 B200 disaggregated TRT-LLM (Dynamo) recipe / Kimi K2.5 NVFP4 B200 TRT-LLM（Dynamo）分离式推理配置 (#2249)
* kimik2.5-fp4-b200-dynamo-trt: Kimi K2.5 NVFP4 B200 disaggregated TRT-LLM (Dynamo) recipe * fix link --------- Co-authored-by: adibarra <93070681+adibarra@users.noreply.github.com>
1 parent c1f3d47 commit a4bb43a

14 files changed

Lines changed: 1607 additions & 0 deletions

‎benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p1d-dep4-dep8-c308-stp.yaml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
name: disagg-B200-1p1d-dep4-dep8-c308-stp
2+
model:
3+
path: kimik2.5-fp4
4+
container: dynamo-trtllm
5+
precision: fp4
6+
identity:
7+
model:
8+
repo: nvidia/Kimi-K2.5-NVFP4
9+
container:
10+
image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21
11+
slurm:
12+
time_limit: 04:00:00
13+
dynamo:
14+
install: true
15+
hash: 4025cb6a8992d19fd19eead36065e08ea5301f35
16+
request_plane: tcp
17+
health_check:
18+
max_attempts: 540
19+
interval_seconds: 10
20+
resources:
21+
gpu_type: b200
22+
gpus_per_node: 8
23+
prefill_nodes: 1
24+
prefill_workers: 1
25+
gpus_per_prefill: 4
26+
decode_nodes: 1
27+
decode_workers: 1
28+
gpus_per_decode: 8
29+
backend:
30+
type: trtllm
31+
prefill_environment:
32+
TLLM_LOG_LEVEL: INFO
33+
TRTLLM_SERVER_DISABLE_GC: '1'
34+
TRTLLM_WORKER_DISABLE_GC: '1'
35+
TRTLLM_ENABLE_PDL: '1'
36+
NCCL_GRAPH_MIXING_SUPPORT: '0'
37+
MIMALLOC_PURGE_DELAY: '0'
38+
UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp
39+
decode_environment:
40+
TLLM_LOG_LEVEL: INFO
41+
TRTLLM_SERVER_DISABLE_GC: '1'
42+
TRTLLM_WORKER_DISABLE_GC: '1'
43+
TRTLLM_ENABLE_PDL: '1'
44+
NCCL_GRAPH_MIXING_SUPPORT: '0'
45+
MIMALLOC_PURGE_DELAY: '0'
46+
UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp
47+
trtllm_config:
48+
prefill:
49+
cache_transceiver_config:
50+
backend: DEFAULT
51+
kv_transfer_timeout_ms: 600000
52+
max_tokens_in_buffer: 68736
53+
cuda_graph_config: null
54+
disable_overlap_scheduler: true
55+
enable_attention_dp: true
56+
kv_cache_config:
57+
dtype: fp8
58+
enable_block_reuse: false
59+
free_gpu_memory_fraction: 0.9
60+
max_batch_size: 2
61+
max_num_tokens: 16386
62+
max_seq_len: 8232
63+
moe_config:
64+
backend: CUTEDSL
65+
moe_expert_parallel_size: 4
66+
nvfp4_gemm_config:
67+
allowed_backends:
68+
- cutlass
69+
- cublaslt
70+
- cutedsl
71+
- cuda_core
72+
pipeline_parallel_size: 1
73+
print_iter_log: true
74+
tensor_parallel_size: 4
75+
trust_remote_code: true
76+
decode:
77+
cache_transceiver_config:
78+
backend: DEFAULT
79+
kv_transfer_timeout_ms: 600000
80+
max_tokens_in_buffer: 68736
81+
cuda_graph_config:
82+
batch_sizes:
83+
- 1
84+
- 2
85+
- 4
86+
- 8
87+
- 16
88+
- 24
89+
- 32
90+
enable_padding: true
91+
enable_attention_dp: true
92+
enable_lm_head_tp_in_adp: false
93+
kv_cache_config:
94+
dtype: fp8
95+
enable_block_reuse: false
96+
free_gpu_memory_fraction: 0.92
97+
max_batch_size: 32
98+
max_num_tokens: 32
99+
max_seq_len: 9256
100+
moe_config:
101+
backend: CUTEDSL
102+
use_low_precision_moe_combine: true
103+
moe_expert_parallel_size: 8
104+
num_postprocess_workers: 4
105+
nvfp4_gemm_config:
106+
allowed_backends:
107+
- cutlass
108+
- cublaslt
109+
- cutedsl
110+
- cuda_core
111+
pipeline_parallel_size: 1
112+
print_iter_log: true
113+
stream_interval: 100
114+
tensor_parallel_size: 8
115+
trust_remote_code: true
116+
frontend:
117+
type: dynamo
118+
enable_multiple_frontends: true
119+
env:
120+
ETCD_LEASE_TTL: '120'
121+
benchmark:
122+
type: sa-bench
123+
isl: 8192
124+
osl: 1024
125+
concurrencies:
126+
- 308
127+
req_rate: inf
128+
num_prompts_mult: 20

‎benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c24-stp.yaml‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
name: disagg-B200-1p4d-dep4-tep8-c24-stp
2+
model:
3+
path: kimik2.5-fp4
4+
container: dynamo-trtllm
5+
precision: fp4
6+
identity:
7+
model:
8+
repo: nvidia/Kimi-K2.5-NVFP4
9+
container:
10+
image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21
11+
slurm:
12+
time_limit: 04:00:00
13+
dynamo:
14+
install: true
15+
hash: 4025cb6a8992d19fd19eead36065e08ea5301f35
16+
request_plane: tcp
17+
health_check:
18+
max_attempts: 540
19+
interval_seconds: 10
20+
resources:
21+
gpu_type: b200
22+
gpus_per_node: 8
23+
prefill_nodes: 1
24+
prefill_workers: 1
25+
gpus_per_prefill: 4
26+
decode_nodes: 4
27+
decode_workers: 4
28+
gpus_per_decode: 8
29+
backend:
30+
type: trtllm
31+
prefill_environment:
32+
TLLM_LOG_LEVEL: INFO
33+
TRTLLM_SERVER_DISABLE_GC: '1'
34+
TRTLLM_WORKER_DISABLE_GC: '1'
35+
TRTLLM_ENABLE_PDL: '1'
36+
NCCL_GRAPH_MIXING_SUPPORT: '0'
37+
MIMALLOC_PURGE_DELAY: '0'
38+
UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp
39+
decode_environment:
40+
TLLM_LOG_LEVEL: INFO
41+
TRTLLM_SERVER_DISABLE_GC: '1'
42+
TRTLLM_WORKER_DISABLE_GC: '1'
43+
TRTLLM_ENABLE_PDL: '1'
44+
NCCL_GRAPH_MIXING_SUPPORT: '0'
45+
MIMALLOC_PURGE_DELAY: '0'
46+
UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp
47+
trtllm_config:
48+
prefill:
49+
cache_transceiver_config:
50+
backend: DEFAULT
51+
kv_transfer_timeout_ms: 600000
52+
max_tokens_in_buffer: 68736
53+
cuda_graph_config: null
54+
disable_overlap_scheduler: true
55+
enable_attention_dp: true
56+
kv_cache_config:
57+
dtype: fp8
58+
enable_block_reuse: false
59+
free_gpu_memory_fraction: 0.9
60+
max_batch_size: 2
61+
max_num_tokens: 16386
62+
max_seq_len: 8232
63+
moe_config:
64+
backend: CUTEDSL
65+
moe_expert_parallel_size: 4
66+
nvfp4_gemm_config:
67+
allowed_backends:
68+
- cutlass
69+
- cublaslt
70+
- cutedsl
71+
- cuda_core
72+
pipeline_parallel_size: 1
73+
print_iter_log: true
74+
tensor_parallel_size: 4
75+
trust_remote_code: true
76+
decode:
77+
cache_transceiver_config:
78+
backend: DEFAULT
79+
kv_transfer_timeout_ms: 600000
80+
max_tokens_in_buffer: 68736
81+
cuda_graph_config:
82+
batch_sizes:
83+
- 1
84+
- 2
85+
- 4
86+
enable_padding: true
87+
enable_attention_dp: false
88+
enable_lm_head_tp_in_adp: false
89+
kv_cache_config:
90+
dtype: fp8
91+
enable_block_reuse: false
92+
free_gpu_memory_fraction: 0.9
93+
max_batch_size: 4
94+
max_num_tokens: 4
95+
max_seq_len: 9256
96+
moe_config:
97+
backend: TRTLLM
98+
use_low_precision_moe_combine: true
99+
moe_expert_parallel_size: 8
100+
num_postprocess_workers: 4
101+
nvfp4_gemm_config:
102+
allowed_backends:
103+
- cutlass
104+
- cublaslt
105+
- cutedsl
106+
- cuda_core
107+
pipeline_parallel_size: 1
108+
print_iter_log: true
109+
stream_interval: 100
110+
tensor_parallel_size: 8
111+
trust_remote_code: true
112+
frontend:
113+
type: dynamo
114+
enable_multiple_frontends: false
115+
env:
116+
ETCD_LEASE_TTL: '120'
117+
benchmark:
118+
type: sa-bench
119+
isl: 8192
120+
osl: 1024
121+
concurrencies:
122+
- 24
123+
req_rate: inf
124+
num_prompts_mult: 20

‎benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c4-stp.yaml‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
name: disagg-B200-1p4d-dep4-tep8-c4-stp
2+
model:
3+
path: kimik2.5-fp4
4+
container: dynamo-trtllm
5+
precision: fp4
6+
identity:
7+
model:
8+
repo: nvidia/Kimi-K2.5-NVFP4
9+
container:
10+
image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21
11+
slurm:
12+
time_limit: 04:00:00
13+
dynamo:
14+
install: true
15+
hash: 4025cb6a8992d19fd19eead36065e08ea5301f35
16+
request_plane: tcp
17+
health_check:
18+
max_attempts: 540
19+
interval_seconds: 10
20+
resources:
21+
gpu_type: b200
22+
gpus_per_node: 8
23+
prefill_nodes: 1
24+
prefill_workers: 1
25+
gpus_per_prefill: 4
26+
decode_nodes: 4
27+
decode_workers: 4
28+
gpus_per_decode: 8
29+
backend:
30+
type: trtllm
31+
prefill_environment:
32+
TLLM_LOG_LEVEL: INFO
33+
TRTLLM_SERVER_DISABLE_GC: '1'
34+
TRTLLM_WORKER_DISABLE_GC: '1'
35+
TRTLLM_ENABLE_PDL: '1'
36+
NCCL_GRAPH_MIXING_SUPPORT: '0'
37+
MIMALLOC_PURGE_DELAY: '0'
38+
UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp
39+
decode_environment:
40+
TLLM_LOG_LEVEL: INFO
41+
TRTLLM_SERVER_DISABLE_GC: '1'
42+
TRTLLM_WORKER_DISABLE_GC: '1'
43+
TRTLLM_ENABLE_PDL: '1'
44+
NCCL_GRAPH_MIXING_SUPPORT: '0'
45+
MIMALLOC_PURGE_DELAY: '0'
46+
UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp
47+
trtllm_config:
48+
prefill:
49+
cache_transceiver_config:
50+
backend: DEFAULT
51+
kv_transfer_timeout_ms: 600000
52+
max_tokens_in_buffer: 68736
53+
cuda_graph_config: null
54+
disable_overlap_scheduler: true
55+
enable_attention_dp: true
56+
kv_cache_config:
57+
dtype: fp8
58+
enable_block_reuse: false
59+
free_gpu_memory_fraction: 0.9
60+
max_batch_size: 2
61+
max_num_tokens: 16386
62+
max_seq_len: 8232
63+
moe_config:
64+
backend: CUTEDSL
65+
moe_expert_parallel_size: 4
66+
nvfp4_gemm_config:
67+
allowed_backends:
68+
- cutlass
69+
- cublaslt
70+
- cutedsl
71+
- cuda_core
72+
pipeline_parallel_size: 1
73+
print_iter_log: true
74+
tensor_parallel_size: 4
75+
trust_remote_code: true
76+
decode:
77+
cache_transceiver_config:
78+
backend: DEFAULT
79+
kv_transfer_timeout_ms: 600000
80+
max_tokens_in_buffer: 68736
81+
cuda_graph_config:
82+
batch_sizes:
83+
- 1
84+
- 2
85+
- 4
86+
enable_padding: true
87+
enable_attention_dp: false
88+
enable_lm_head_tp_in_adp: false
89+
kv_cache_config:
90+
dtype: fp8
91+
enable_block_reuse: false
92+
free_gpu_memory_fraction: 0.9
93+
max_batch_size: 1
94+
max_num_tokens: 1
95+
max_seq_len: 9256
96+
moe_config:
97+
backend: TRTLLM
98+
use_low_precision_moe_combine: true
99+
moe_expert_parallel_size: 8
100+
num_postprocess_workers: 4
101+
nvfp4_gemm_config:
102+
allowed_backends:
103+
- cutlass
104+
- cublaslt
105+
- cutedsl
106+
- cuda_core
107+
pipeline_parallel_size: 1
108+
print_iter_log: true
109+
stream_interval: 100
110+
tensor_parallel_size: 8
111+
trust_remote_code: true
112+
frontend:
113+
type: dynamo
114+
enable_multiple_frontends: false
115+
env:
116+
ETCD_LEASE_TTL: '120'
117+
benchmark:
118+
type: sa-bench
119+
isl: 8192
120+
osl: 1024
121+
concurrencies:
122+
- 4
123+
req_rate: inf
124+
num_prompts_mult: 20

0 commit comments

Comments
 (0)