diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index bd204ab36..34a422f1c 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -2632,7 +2632,7 @@ kimik2.5-int4-h200-vllm-agentic: - { tp: 8, offloading: cpu, conc-list: [6, 7, 8, 9, 10, 11, 12, 13, 14] } kimik2.5-fp4-b200-vllm: - image: vllm/vllm-openai:v0.17.0 + image: vllm/vllm-openai:v0.20.2 model: nvidia/Kimi-K2.5-NVFP4 model-prefix: kimik2.5 runner: b200 diff --git a/benchmarks/single_node/kimik2.5_fp4_b200.sh b/benchmarks/single_node/kimik2.5_fp4_b200.sh index 9c82d2865..b4e85c14f 100644 --- a/benchmarks/single_node/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/kimik2.5_fp4_b200.sh @@ -33,6 +33,13 @@ fi # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor +# vLLM v0.20.2+'s CUDA-graph memory profiler pre-reserves ~57 GB/GPU upfront +# (~32% of total), which collides with --gpu-memory-utilization=0.90 and +# leaves negative space for the KV cache. Disable the profiler — our 0.90 +# already leaves ~18 GB/GPU as safety net (same pattern as +# benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh). +export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 + set -x vllm serve $MODEL --host 0.0.0.0 --port $PORT \ --tensor-parallel-size=$TP \ diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d053e65c5..a28c66b85 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -2704,3 +2704,10 @@ description: - "Update vLLM image from v0.19.0-cu130 (25d old) to v0.21.0" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1448 + +- config-keys: + - kimik2.5-fp4-b200-vllm + description: + - "Update vLLM image from v0.20.2 to v0.21.0" + - "Add VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 to disable aggressive CUDA-graph memory profiler that OOMs the KV cache" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1395