[https://nvbugs/6108994][fix] add kv_transfer_timeout_ms to avoid timeout (#15152)

Signed-off-by: Bo Deng <deemod@nvidia.com>

Author: bo-nv

tests/integration/test_lists/waives.txt

@@ -322,10 +322,8 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1
 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6215844)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6280649)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6179661)
-perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6280649)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6280649)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6280649)
-perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6221024)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] SKIP (https://nvbugs/6280649)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6280721)
 perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6280649)

tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml

@@ -63,6 +63,7 @@ worker_config:
     cache_transceiver_config:
       max_tokens_in_buffer: 131104
       backend: NIXL
+      kv_transfer_timeout_ms: 600000
     disable_overlap_scheduler: true
     speculative_config: &id001
       decoding_type: MTP
@@ -89,5 +90,6 @@ worker_config:
     cache_transceiver_config:
       max_tokens_in_buffer: 131104
       backend: NIXL
+      kv_transfer_timeout_ms: 600000
     disable_overlap_scheduler: true
     speculative_config: *id001

tests/scripts/perf-sanity/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml

@@ -70,6 +70,7 @@ worker_config:
     cache_transceiver_config:
       max_tokens_in_buffer: 120000
       backend: NIXL
+      kv_transfer_timeout_ms: 600000
     num_postprocess_workers: 8
     stream_interval: 10
   ctx:
@@ -97,3 +98,4 @@ worker_config:
     cache_transceiver_config:
       max_tokens_in_buffer: 120000
       backend: NIXL
+      kv_transfer_timeout_ms: 600000

tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml

@@ -74,6 +74,7 @@ worker_config:
     cache_transceiver_config:
       max_tokens_in_buffer: 131104
       backend: NIXL
+      kv_transfer_timeout_ms: 600000
     disable_overlap_scheduler: true
     speculative_config: &id001
       decoding_type: MTP
@@ -100,5 +101,6 @@ worker_config:
     cache_transceiver_config:
       max_tokens_in_buffer: 131104
       backend: NIXL
+      kv_transfer_timeout_ms: 600000
     disable_overlap_scheduler: true
     speculative_config: *id001

tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml

@@ -75,6 +75,7 @@ worker_config:
     cache_transceiver_config:
       max_tokens_in_buffer: 131104
       backend: NIXL
+      kv_transfer_timeout_ms: 600000
     disable_overlap_scheduler: true
     speculative_config: &id001
       decoding_type: MTP
@@ -101,5 +102,6 @@ worker_config:
     cache_transceiver_config:
       max_tokens_in_buffer: 131104
       backend: NIXL
+      kv_transfer_timeout_ms: 600000
     disable_overlap_scheduler: true
     speculative_config: *id001

tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-UCX.yaml

@@ -75,6 +75,7 @@ worker_config:
     cache_transceiver_config:
       max_tokens_in_buffer: 131104
       backend: UCX
+      kv_transfer_timeout_ms: 600000
     disable_overlap_scheduler: true
     speculative_config: &id001
       decoding_type: MTP
@@ -101,5 +102,6 @@ worker_config:
     cache_transceiver_config:
       max_tokens_in_buffer: 131104
       backend: UCX
+      kv_transfer_timeout_ms: 600000
     disable_overlap_scheduler: true
     speculative_config: *id001

tests/scripts/perf/disaggregated/gb300_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml

@@ -70,6 +70,7 @@ worker_config:
     cache_transceiver_config:
       max_tokens_in_buffer: 120000
       backend: NIXL
+      kv_transfer_timeout_ms: 600000
     num_postprocess_workers: 8
     stream_interval: 10
   ctx:
@@ -97,3 +98,4 @@ worker_config:
     cache_transceiver_config:
       max_tokens_in_buffer: 120000
       backend: NIXL
+      kv_transfer_timeout_ms: 600000