Skip to content

Commit ce67288

Browse files
nv-xtfnv-xtf
authored
[None][perf] Validate GPT-OSS transceiver v2 performance (NVIDIA#15765)
Signed-off-by: Tingfeng Xian <289617005+nv-xtf@users.noreply.github.com>
1 parent dc49938 commit ce67288

20 files changed

Lines changed: 80 additions & 10 deletions

File tree

‎tests/integration/defs/accuracy/test_disaggregated_serving.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1570,15 +1570,17 @@ def test_auto_dtype(self, block_reuse, mocker):
15701570
ctx_server_config = {
15711571
"disable_overlap_scheduler": True,
15721572
"cache_transceiver_config": {
1573-
"backend": "DEFAULT",
1573+
"backend": "NIXL",
1574+
"transceiver_runtime": "PYTHON",
15741575
"max_tokens_in_buffer": 4096
15751576
},
15761577
"tensor_parallel_size": 4
15771578
}
15781579
gen_server_config = {
15791580
"disable_overlap_scheduler": False,
15801581
"cache_transceiver_config": {
1581-
"backend": "DEFAULT",
1582+
"backend": "NIXL",
1583+
"transceiver_runtime": "PYTHON",
15821584
"max_tokens_in_buffer": 4096
15831585
},
15841586
"tensor_parallel_size": 4

‎tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_triton.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ context_servers:
2626
cuda_graph_config: null
2727
print_iter_log: true
2828
cache_transceiver_config:
29-
backend: DEFAULT
29+
backend: NIXL
30+
transceiver_runtime: PYTHON
3031
max_tokens_in_buffer: 16384
3132
generation_servers:
3233
num_instances: 1
@@ -63,5 +64,6 @@ generation_servers:
6364
- 1024
6465
print_iter_log: true
6566
cache_transceiver_config:
66-
backend: DEFAULT
67+
backend: NIXL
68+
transceiver_runtime: PYTHON
6769
max_tokens_in_buffer: 16384

‎tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_eagle_trtllm.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ context_servers:
2626
cuda_graph_config: null
2727
print_iter_log: true
2828
cache_transceiver_config:
29-
backend: DEFAULT
29+
backend: NIXL
30+
transceiver_runtime: PYTHON
3031
max_tokens_in_buffer: 16384
3132
generation_servers:
3233
num_instances: 1
@@ -63,5 +64,6 @@ generation_servers:
6364
- 1024
6465
print_iter_log: true
6566
cache_transceiver_config:
66-
backend: DEFAULT
67+
backend: NIXL
68+
transceiver_runtime: PYTHON
6769
max_tokens_in_buffer: 16384

‎tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_tllm.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ context_servers:
2222
cuda_graph_config: null
2323
print_iter_log: true
2424
cache_transceiver_config:
25-
backend: DEFAULT
25+
backend: NIXL
26+
transceiver_runtime: PYTHON
2627
max_tokens_in_buffer: 16384
2728
generation_servers:
2829
num_instances: 1
@@ -59,5 +60,6 @@ generation_servers:
5960
- 1024
6061
print_iter_log: true
6162
cache_transceiver_config:
62-
backend: DEFAULT
63+
backend: NIXL
64+
transceiver_runtime: PYTHON
6365
max_tokens_in_buffer: 16384

‎tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_gptoss_triton.yaml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ context_servers:
2121
cuda_graph_config: null
2222
print_iter_log: true
2323
cache_transceiver_config:
24-
backend: DEFAULT
24+
backend: NIXL
25+
transceiver_runtime: PYTHON
2526
max_tokens_in_buffer: 16384
2627
generation_servers:
2728
num_instances: 1
@@ -53,5 +54,6 @@ generation_servers:
5354
- 128
5455
print_iter_log: true
5556
cache_transceiver_config:
56-
backend: DEFAULT
57+
backend: NIXL
58+
transceiver_runtime: PYTHON
5759
max_tokens_in_buffer: 16384

‎tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con2048_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ worker_config:
5757
enable_padding: true
5858
max_batch_size: 1536
5959
kv_cache_config:
60+
use_kv_cache_manager_v2: false
6061
enable_block_reuse: false
6162
free_gpu_memory_fraction: 0.9
6263
dtype: fp8
@@ -65,6 +66,7 @@ worker_config:
6566
cache_transceiver_config:
6667
max_tokens_in_buffer: 1024
6768
backend: NIXL
69+
transceiver_runtime: PYTHON
6870
disable_overlap_scheduler: true
6971
num_postprocess_workers: 4
7072
stream_interval: 20
@@ -79,6 +81,7 @@ worker_config:
7981
enable_attention_dp: false
8082
cuda_graph_config: null
8183
kv_cache_config:
84+
use_kv_cache_manager_v2: false
8285
enable_block_reuse: false
8386
free_gpu_memory_fraction: 0.9
8487
dtype: fp8
@@ -88,4 +91,5 @@ worker_config:
8891
cache_transceiver_config:
8992
max_tokens_in_buffer: 1024
9093
backend: NIXL
94+
transceiver_runtime: PYTHON
9195
disable_overlap_scheduler: true

‎tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con512_ctx1_tp1_gen1_dep2_eplb0_mtp0_ccb-NIXL.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ worker_config:
5757
enable_padding: true
5858
max_batch_size: 1536
5959
kv_cache_config:
60+
use_kv_cache_manager_v2: false
6061
enable_block_reuse: false
6162
free_gpu_memory_fraction: 0.9
6263
dtype: fp8
@@ -65,6 +66,7 @@ worker_config:
6566
cache_transceiver_config:
6667
max_tokens_in_buffer: 1024
6768
backend: NIXL
69+
transceiver_runtime: PYTHON
6870
disable_overlap_scheduler: true
6971
num_postprocess_workers: 4
7072
stream_interval: 20
@@ -79,6 +81,7 @@ worker_config:
7981
enable_attention_dp: false
8082
cuda_graph_config: null
8183
kv_cache_config:
84+
use_kv_cache_manager_v2: false
8285
enable_block_reuse: false
8386
free_gpu_memory_fraction: 0.9
8487
dtype: fp8
@@ -88,4 +91,5 @@ worker_config:
8891
cache_transceiver_config:
8992
max_tokens_in_buffer: 1024
9093
backend: NIXL
94+
transceiver_runtime: PYTHON
9195
disable_overlap_scheduler: true

‎tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ worker_config:
5353
enable_padding: true
5454
max_batch_size: 256
5555
kv_cache_config:
56+
use_kv_cache_manager_v2: false
5657
enable_block_reuse: false
5758
free_gpu_memory_fraction: 0.9
5859
dtype: fp8
@@ -61,6 +62,7 @@ worker_config:
6162
cache_transceiver_config:
6263
max_tokens_in_buffer: 1024
6364
backend: NIXL
65+
transceiver_runtime: PYTHON
6466
disable_overlap_scheduler: true
6567
num_postprocess_workers: 4
6668
stream_interval: 20
@@ -75,6 +77,7 @@ worker_config:
7577
enable_attention_dp: false
7678
cuda_graph_config: null
7779
kv_cache_config:
80+
use_kv_cache_manager_v2: false
7881
enable_block_reuse: false
7982
free_gpu_memory_fraction: 0.9
8083
dtype: fp8
@@ -84,4 +87,5 @@ worker_config:
8487
cache_transceiver_config:
8588
max_tokens_in_buffer: 1024
8689
backend: NIXL
90+
transceiver_runtime: PYTHON
8791
disable_overlap_scheduler: true

‎tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ worker_config:
6565
enable_padding: true
6666
max_batch_size: 1280
6767
kv_cache_config:
68+
use_kv_cache_manager_v2: false
6869
enable_block_reuse: false
6970
free_gpu_memory_fraction: 0.85
7071
dtype: fp8
@@ -73,6 +74,7 @@ worker_config:
7374
cache_transceiver_config:
7475
max_tokens_in_buffer: 9216
7576
backend: NIXL
77+
transceiver_runtime: PYTHON
7678
disable_overlap_scheduler: false
7779
num_postprocess_workers: 4
7880
stream_interval: 20
@@ -91,6 +93,7 @@ worker_config:
9193
enable_padding: true
9294
max_batch_size: 30
9395
kv_cache_config:
96+
use_kv_cache_manager_v2: false
9497
enable_block_reuse: false
9598
free_gpu_memory_fraction: 0.8
9699
dtype: fp8
@@ -99,6 +102,7 @@ worker_config:
99102
cache_transceiver_config:
100103
max_tokens_in_buffer: 9216
101104
backend: NIXL
105+
transceiver_runtime: PYTHON
102106
disable_overlap_scheduler: true
103107
num_postprocess_workers: 4
104108
stream_interval: 20

‎tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ worker_config:
5353
enable_padding: true
5454
max_batch_size: 1024
5555
kv_cache_config:
56+
use_kv_cache_manager_v2: false
5657
enable_block_reuse: false
5758
free_gpu_memory_fraction: 0.9
5859
dtype: fp8
@@ -61,6 +62,7 @@ worker_config:
6162
cache_transceiver_config:
6263
max_tokens_in_buffer: 8448
6364
backend: NIXL
65+
transceiver_runtime: PYTHON
6466
disable_overlap_scheduler: true
6567
num_postprocess_workers: 4
6668
stream_interval: 20
@@ -75,6 +77,7 @@ worker_config:
7577
enable_attention_dp: false
7678
cuda_graph_config: null
7779
kv_cache_config:
80+
use_kv_cache_manager_v2: false
7881
enable_block_reuse: false
7982
free_gpu_memory_fraction: 0.9
8083
dtype: fp8
@@ -84,4 +87,5 @@ worker_config:
8487
cache_transceiver_config:
8588
max_tokens_in_buffer: 8448
8689
backend: NIXL
90+
transceiver_runtime: PYTHON
8791
disable_overlap_scheduler: true

0 commit comments

Comments
 (0)