[AgentX] vLLM DeepSeek-V4 B200 aggregate / vLLM DeepSeek-V4 B200 聚合配置 (#2224)

* feat(agentic): split DeepSeek-V4 B200 vLLM recipe

Carry the B200-only launcher, config search space, and changelog scope from PR #2202.

中文：拆分 DeepSeek-V4 B200 vLLM AgentX 配方，仅保留 PR #2202 中的 B200 启动器、配置搜索空间和变更日志范围。

* docs(changelog): link B200 replacement PR

Point the split B200 changelog entry to PR #2224.

中文：将拆分后的 B200 变更日志条目链接到 PR #2224。

* chore: fix mooncake dev and conc

* fix: terminate perf changelog with newline

* fix: keep perf changelog append-only

---------

Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>

Author: cquil11

benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
 set -x
 
 # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B200 using vLLM.
@@ -14,21 +14,25 @@ set -x
 #       Highest aggregate throughput at large CONC.
 #
 # Image is configured in nvidia-master.yaml. block_size=256,
-# kv-cache-dtype=fp8, FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph
-# capture with custom_ops=all (per the vLLM blog recipe at
-# https://vllm.ai/blog/deepseek-v4).
+# kv-cache-dtype=fp8, FLASHINFER_MLA_SPARSE_DSV4 attention with the FP4 indexer
+# cache, FULL_DECODE_ONLY cudagraph capture, and (in EP tiers) mega-MoE backend.
 #
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake.
+# Pure TP is GPU-resident (KV_OFFLOADING=none). DEP tiers offload KV to host
+# DRAM: KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=vllm-simple or mooncake.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
 check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
 
-DCP_SIZE="${DCP_SIZE:-1}"
-PCP_SIZE="${PCP_SIZE:-1}"
+if [ -z "$DCP_SIZE" ]; then
+    DCP_SIZE=1
+fi
+if [ -z "$PCP_SIZE" ]; then
+    PCP_SIZE=1
+fi
 VLLM_CP_ARGS=()
 if [ "$DCP_SIZE" -gt 1 ]; then
     VLLM_CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE")
@@ -37,21 +41,21 @@ if [ "$PCP_SIZE" -gt 1 ]; then
     VLLM_CP_ARGS+=(--prefill-context-parallel-size "$PCP_SIZE")
 fi
 
-GPU_COUNT="${GPU_COUNT:-$((TP * PCP_SIZE))}"
+GPU_COUNT=$TP
 if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then
     echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2
     exit 1
 fi
 export GPU_COUNT
 
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
-    echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
+if [[ -n "$SLURM_JOB_ID" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
 fi
 
 # `hf download` creates the target dir if missing and is itself idempotent.
 # When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
 # Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
+if [[ -n "$MODEL_PATH" ]]; then
     if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
         hf download "$MODEL" --local-dir "$MODEL_PATH"
     fi
@@ -65,12 +69,6 @@ nvidia-smi
 resolve_trace_source
 install_agentic_deps
 
-# vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings,
-# which fails DSV4 indexer compilation with mlir_global_dtors(..., data).
-# Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved.
-agentic_pip_install --quiet --force-reinstall --no-deps \
-    'nvidia-cutlass-dsl-libs-cu13==4.5.2'
-
 # vllm-project/router expands the one HTTP backend into one logical worker per
 # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's
 # X-Correlation-ID is stable for every turn of a conversation; alias it to the
@@ -94,6 +92,10 @@ export VLLM_ENGINE_READY_TIMEOUT_S=3600
 # vllm-project/vllm#44774 applies the same reachability policy to Mooncake's
 # store mask. 32k matches the trace-replay tuning validated for this workload.
 export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768
+export VLLM_USE_V2_MODEL_RUNNER=1
+export VLLM_USE_RUST_FRONTEND=1
+export VLLM_DSV4_MEGA_FP8_COMBINE=1
+export VLLM_RPC_TIMEOUT=600000
 
 # ---- Server config ----------------------------------------------------------
 SERVER_LOG="$RESULT_DIR/server.log"
@@ -106,8 +108,34 @@ ROUTER_PID=""
 MOONCAKE_MASTER_PID=""
 
 OFFLOAD_ARGS=()
-
-if require_agentic_kv_offload_backend mooncake; then
+case "$KV_OFFLOAD_BACKEND" in
+    "")
+        require_agentic_kv_offload_none
+        ;;
+    vllm-simple)
+        require_agentic_kv_offload_backend vllm-simple
+        CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT ))
+        # Identical prefixes must hash to identical block keys across DP ranks.
+        export PYTHONHASHSEED=42
+        OFFLOAD_CONFIG=$(cat <<EOF
+{
+  "kv_connector": "SimpleCPUOffloadConnector",
+  "kv_role": "kv_both",
+  "kv_connector_extra_config": {
+    "cpu_bytes_to_use_per_rank": ${CPU_BYTES_PER_RANK},
+    "lazy_offload": false,
+    "enable_cross_layers_blocks": "true"
+  }
+}
+EOF
+)
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "$OFFLOAD_CONFIG"
+        )
+        ;;
+    mooncake)
+        require_agentic_kv_offload_backend mooncake
         # Embedded mode contributes one segment per GPU rank to a shared
         # distributed store, so pre-divide the aggregate host-memory budget.
         PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT))
@@ -127,16 +155,14 @@ if require_agentic_kv_offload_backend mooncake; then
   "global_segment_size": "${PER_RANK_GB}GB",
   "local_buffer_size": "4GB",
   "protocol": "rdma",
-  "device_name": "mlx5_0",
+  "device_name": "mlx5_0,mlx5_1,mlx5_2,mlx5_3,mlx5_4,mlx5_5,mlx5_10,mlx5_11",
   "enable_offload": false
 }
 EOF
         export MOONCAKE_CONFIG_PATH
+        export MC_ENABLE_DEST_DEVICE_AFFINITY=1
         # Identical prefixes must hash to identical store keys across DP ranks.
         export PYTHONHASHSEED=0
-        # B200 GPU memory registration works through DMA-BUF, but the compute
-        # nodes do not expose nvidia_peermem. Force Mooncake's DMA-BUF
-        # GPUDirect RDMA path instead of its legacy ibv_reg_mr path.
         export WITH_NVIDIA_PEERMEM=0
         export MC_SLICE_SIZE=1048576
         export MC_WORKERS_PER_CTX=4
@@ -167,16 +193,27 @@ EOF
             --kv-transfer-config
             '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
         )
-fi
+        ;;
+    *)
+        echo "Error: unsupported B200 KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2
+        exit 1
+        ;;
+esac
 
 PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
 if [ "$DP_ATTENTION" = "true" ]; then
     PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
 fi
 
 EP_ARGS=()
+FAST_MOE_ARGS=()
 if [ "$EP_SIZE" -gt 1 ]; then
     EP_ARGS=(--enable-expert-parallel)
+    FAST_MOE_ARGS=(
+        --moe-backend deep_gemm_amxf4_mega_moe
+        --enable-ep-weight-filter
+        --prefill-schedule-interval 16
+    )
 fi
 
 # AgentX concurrency counts live session trees, not individual requests.
@@ -197,18 +234,23 @@ VLLM_CMD=(
     --trust-remote-code
     --kv-cache-dtype fp8
     --block-size 256
-    "${PARALLEL_ARGS[@]}"
-    "${VLLM_CP_ARGS[@]}"
-    "${EP_ARGS[@]}"
-    --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
-    --attention_config.use_fp4_indexer_cache=True
+    --max-model-len 1048576
+    --gpu-memory-utilization 0.92
+    --numa-bind
+    --enable-cumem-allocator
+    --no-enable-flashinfer-autotune
     --tokenizer-mode deepseek_v4
-    --tool-call-parser deepseek_v4
-    --enable-auto-tool-choice
     --reasoning-parser deepseek_v4
-    --enable-prefix-caching
+    --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
     --no-disable-hybrid-kv-cache-manager
+    --disable-uvicorn-access-log
+    --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
     --max-num-seqs "$MAX_NUM_SEQS"
+    --max-cudagraph-capture-size "$MAX_NUM_SEQS"
+    "${PARALLEL_ARGS[@]}"
+    "${VLLM_CP_ARGS[@]}"
+    "${EP_ARGS[@]}"
+    "${FAST_MOE_ARGS[@]}"
     "${OFFLOAD_ARGS[@]}"
 )
 printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"

configs/nvidia-master.yaml

@@ -1771,7 +1771,7 @@ dsv4-fp4-b200-vllm:
       - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 }
 
 dsv4-fp4-b200-vllm-agentic:
-  image: vllm/vllm-openai:v0.23.0
+  image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:b200-dgxc
@@ -1782,14 +1782,12 @@ dsv4-fp4-b200-vllm-agentic:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      # Pure TP is only competitive at very low concurrency.
-      - { tp: 8, kv-offloading: none,  conc-list: [1, 2, 3, 4, 5] }
-      # Sample the useful MooncakeStore range without repeating its collapsed
-      # high-concurrency tail.
-      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [8, 10, 16] }
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none,  conc-list: [16, 32, 38, 44, 50], router: { name: vllm-router, version: "0.1.14" } }
-      # Retain the external-cache transition and peak-throughput region.
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [16, 38, 44, 56, 64, 66, 68], router: { name: vllm-router, version: "0.1.14" } }
+      # Pure TP at low concurrency.
+      - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8] }
+      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 12, 16] }
+      # DEP
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 68, 72, 80], router: { name: vllm-router, version: "0.1.14" } }
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b200-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
@@ -3346,7 +3344,7 @@ dsr1-fp8-b200-trt:
       osl: 1024
       search-space:
       - { tp: 8, ep: 1, conc-start: 64, conc-end: 128 }
-      - { tp: 4, ep: 1, conc-start: 8, conc-end: 16 } 
+      - { tp: 4, ep: 1, conc-start: 8, conc-end: 16 }
       - { tp: 8, ep: 1, conc-start: 4, conc-end: 8 }
     - isl: 8192
       osl: 1024

perf-changelog.yaml

@@ -4887,3 +4887,10 @@
     - "Add SGLANG_MAMBA_SSM_DTYPE=bfloat16 in both non-MTP and MTP benchmark scripts"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201  
 
+- config-keys:
+    - dsv4-fp4-b200-vllm-agentic
+  description:
+    - "Update B200 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
+    - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
+    - "B200: GPU-resident TP8 at conc [1,2,4,6,8] and SimpleCPU-offload TP8 at conc [8,12,16]; DEP8 at conc [8,16,24,32,40,48,56,64,68,72] with both SimpleCPU and Mooncake 0.3.11.post1."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2224