[AgentX] Update vLLM DeepSeek-V4 B300 aggregate / 更新 vLLM DeepSeek-V4 B300 聚合配置 (#2241)

ivanium · cquil11 · majunze2001 · web-flow · commit c31c9749a32b · 2026-07-16T16:56:22.000-05:00
* feat(agentic): split DeepSeek-V4 B300 vLLM recipe Carry the B300-only launcher, config search space, and changelog scope from PR #2202. 中文：拆分 DeepSeek-V4 B300 vLLM AgentX 配方，仅保留 PR #2202 中的 B300 启动器、配置搜索空间和变更日志范围。 * docs(changelog): link B300 replacement PR Point the split B300 changelog entry to PR #2225. 中文：将拆分后的 B300 变更日志条目链接到 PR #2225。 * chore: update conc * update dep8 args Signed-off-by: Jeff Ma <jeffjma@umich.edu> * add more tp configs Signed-off-by: Jeff Ma <jeffjma@umich.edu> * dep8 reduce gpu mem util to 0.92 Signed-off-by: Jeff Ma <jeffjma@umich.edu> * chore: update conc list * docs(changelog): point B300 entry to PR #2241 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LjHXfE1FRN9c5XQqgeTdez * fix(changelog): keep perf-changelog additions-only Restore upstream lines whitespace-for-whitespace and append the B300 entry so the changelog diff contains no deletions. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LjHXfE1FRN9c5XQqgeTdez --------- Signed-off-by: Jeff Ma <jeffjma@umich.edu> Co-authored-by: Cam Quilici <cjquilici@gmail.com> Co-authored-by: Jeff Ma <jeffjma@umich.edu> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh
@@ -1,60 +1,54 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -eo pipefail
 set -x
 
 # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using vLLM.
-# Mirrors the fixed-seq-len parallelism options (pure TP and DEP) so the
-# agentic sweep can probe both interactivity and throughput regimes:
-#   pure TP (DP_ATTENTION=false, EP_SIZE=1):  attention TP-sharded across
-#       all $TP GPUs in a single engine. Lower TPOT, lower batch.
-#   TP+EP   (DP_ATTENTION=false, EP_SIZE>1):  attention TP-sharded, MoE
-#       experts EP-sharded within the TP group.
-#   DEP     (DP_ATTENTION=true, EP_SIZE>1):   per-DP-rank attention with
-#       experts EP-sharded across DP ranks (per the vLLM blog recipe).
-#       Highest aggregate throughput at large CONC.
+# v4pro-b300.yaml TP4, DEP4, and DEP8 recipe. SimpleCPUOffload / MooncakeStore
 #
-# Image is vllm/vllm-openai:v0.20.0-cu130. block_size=256, kv-cache-dtype=fp8,
-# FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph capture with
-# custom_ops=all (per the vLLM blog recipe at https://vllm.ai/blog/deepseek-v4).
+# Image is configured in nvidia-master.yaml. The recipe uses FP8 KV cache,
+# sparse DeepSeek-V4 FlashInfer attention with an FP4 indexer cache, mega-MoE,
+# and FULL_DECODE_ONLY CUDA graphs with every batch size captured explicitly.
 #
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake.
+# TP4, TP8, and DEP8 (TP8 + DP-attention) are GPU-resident (KV_OFFLOADING=none).
+# DEP4 uses KV_OFFLOADING=dram with KV_OFFLOAD_BACKEND=vllm-simple or mooncake.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
 check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
 
-DCP_SIZE="${DCP_SIZE:-1}"
-PCP_SIZE="${PCP_SIZE:-1}"
-VLLM_CP_ARGS=()
-if [ "$DCP_SIZE" -gt 1 ]; then
-    VLLM_CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE")
-fi
-if [ "$PCP_SIZE" -gt 1 ]; then
-    VLLM_CP_ARGS+=(--prefill-context-parallel-size "$PCP_SIZE")
-fi
-
-GPU_COUNT="${GPU_COUNT:-$((TP * PCP_SIZE))}"
+GPU_COUNT=$TP
 if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then
     echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2
     exit 1
 fi
 export GPU_COUNT
 
-if declare -p SLURM_JOB_ID >/dev/null 2>&1 && [ -n "$SLURM_JOB_ID" ]; then
-    SLURM_NODE=unknown
-    if declare -p SLURMD_NODENAME >/dev/null 2>&1 && [ -n "$SLURMD_NODENAME" ]; then
-        SLURM_NODE="$SLURMD_NODENAME"
-    fi
-    echo "JOB $SLURM_JOB_ID running on $SLURM_NODE"
+# Under DP-attention the DP world size equals TP, and the DEP recipe sizes
+# per-rank batch as MAX_NUM_SEQS = 2*CONC/TP, which must be an integer.
+if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then
+    echo "Error: DEP requires 2*CONC divisible by TP, got CONC='$CONC' and TP='$TP'" >&2
+    exit 1
+fi
+
+# DEP8 (TP8 + DP-attention) is a GPU-resident, high-concurrency arm that is
+# tuned separately from the smaller DEP4 arm (larger prefill token budget,
+# long-prefill chunking, and a lower GPU-memory-utilization headroom).
+IS_DEP8=false
+if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then
+    IS_DEP8=true
+fi
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+    echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
 fi
 
 # `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
+# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE.
 # Either way, MODEL_PATH is what the server is launched with.
-if declare -p MODEL_PATH >/dev/null 2>&1 && [ -n "$MODEL_PATH" ]; then
+if [[ -n "$MODEL_PATH" ]]; then
     if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
         hf download "$MODEL" --local-dir "$MODEL_PATH"
     fi
@@ -68,17 +62,9 @@ nvidia-smi
 resolve_trace_source
 install_agentic_deps
 
-# vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings,
-# which fails DSV4 indexer compilation with mlir_global_dtors(..., data).
-# Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved.
-agentic_pip_install --quiet --force-reinstall --no-deps \
-    'nvidia-cutlass-dsl-libs-cu13==4.5.2'
-
 # vllm-project/router expands the one HTTP backend into one logical worker per
-# DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's
-# X-Correlation-ID is stable for every turn of a conversation; alias it to the
-# router's preferred X-Session-ID header. This also keeps affinity correct when
-# testing older wheels that prioritize per-request X-Request-ID.
+# DP rank. Bind every turn of a conversation to the same rank by mapping
+# AIPerf's stable correlation ID to the router's X-Session-ID header.
 USE_VLLM_ROUTER=false
 VLLM_BACKEND_PORT="$PORT"
 if [ "$DP_ATTENTION" = "true" ]; then
@@ -91,13 +77,13 @@ if [ "$DP_ATTENTION" = "true" ]; then
     agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION"
 fi
 
-# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s.
+# Match the environment used by v4pro-b300.yaml.
+export VLLM_USE_V2_MODEL_RUNNER=1
 export VLLM_ENGINE_READY_TIMEOUT_S=3600
-
-# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while
-# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's
-# store mask. 32k matches the trace-replay tuning validated for this workload.
 export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768
+export VLLM_DSV4_MEGA_FP8_COMBINE=1
+export NCCL_NVLS_ENABLE=1
+export VLLM_USE_RUST_FRONTEND=1
 
 # ---- Server config ----------------------------------------------------------
 SERVER_LOG="$RESULT_DIR/server.log"
@@ -109,13 +95,47 @@ SERVER_PID=""
 ROUTER_PID=""
 MOONCAKE_MASTER_PID=""
 
+# The generated TOTAL_CPU_DRAM_GB budget is proportional to allocated GPUs.
+# On cluster:b300-nv, dram-utilization=0.80 and DEP4 resolve to roughly the
+# source recipe's 280 GiB per DP rank. TP4 remains GPU-resident.
 OFFLOAD_ARGS=()
-if require_agentic_kv_offload_backend mooncake; then
-        # Mooncake embedded mode contributes one global segment per GPU rank to
-        # a shared distributed store. Pre-divide the aggregate host budget
-        # across those rank-contributed segments.
+case "$KV_OFFLOAD_BACKEND" in
+    "")
+        require_agentic_kv_offload_none
+        ;;
+    vllm-simple)
+        require_agentic_kv_offload_backend vllm-simple
+        CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT ))
+        # Identical prefixes must hash to identical block keys across DP ranks.
+        export PYTHONHASHSEED=42
+        # The plain-TP (non-DP-attention) offload ladder uses lazy offload;
+        # DEP keeps eager offload for cross-rank block-hash stability.
+        SIMPLE_LAZY_OFFLOAD=false
+        if [ "$DP_ATTENTION" != "true" ]; then
+            SIMPLE_LAZY_OFFLOAD=true
+        fi
+        OFFLOAD_CONFIG=$(cat <<EOF
+{
+  "kv_connector": "SimpleCPUOffloadConnector",
+  "kv_role": "kv_both",
+  "kv_connector_extra_config": {
+    "cpu_bytes_to_use": ${CPU_BYTES_PER_RANK},
+    "enable_cross_layers_blocks": "true",
+    "lazy_offload": ${SIMPLE_LAZY_OFFLOAD}
+  }
+}
+EOF
+)
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "$OFFLOAD_CONFIG"
+        )
+        ;;
+    mooncake)
+        require_agentic_kv_offload_backend mooncake
+        # Embedded mode contributes one global segment per DP rank to the
+        # shared store, so divide the aggregate host budget across ranks.
         PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / GPU_COUNT))
-
         MOONCAKE_VERSION=0.3.11.post1
         agentic_pip_install --quiet --no-cache-dir --no-deps \
             --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
@@ -139,9 +159,7 @@ EOF
         export MC_ENABLE_DEST_DEVICE_AFFINITY=1
         # Identical prefixes must hash to identical store keys across DP ranks.
         export PYTHONHASHSEED=0
-        # Large agentic KV writes can exceed Mooncake Store's fixed 60-second
-        # transfer deadline at the default 64 KiB RDMA slice size. Reduce
-        # per-transfer bookkeeping and give the shared RNIC more workers.
+        export WITH_NVIDIA_PEERMEM=0
         export MC_SLICE_SIZE=1048576
         export MC_WORKERS_PER_CTX=4
 
@@ -165,54 +183,106 @@ EOF
         fi
 
         unset VLLM_USE_SIMPLE_KV_OFFLOAD
-        OFFLOAD_ARGS=(
-            --kv-transfer-config
-            '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
-        )
-fi
+        OFFLOAD_CONFIG='{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
+        OFFLOAD_ARGS=(--kv-transfer-config "$OFFLOAD_CONFIG")
+        ;;
+    *)
+        echo "Error: unsupported B300 KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2
+        exit 1
+        ;;
+esac
 
 PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
 if [ "$DP_ATTENTION" = "true" ]; then
     PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
 fi
 
-EP_ARGS=()
+TP_ARGS=()
+if [ "$DP_ATTENTION" = "true" ]; then
+    export PYTORCH_ALLOC_CONF=expandable_segments:True
+else
+    export VLLM_ALLREDUCE_USE_FLASHINFER=1
+    export VLLM_FLASHINFER_ALLREDUCE_BACKEND=auto
+    TP_ARGS+=(--disable-custom-all-reduce)
+fi
+
+MODE_ARGS=()
 if [ "$EP_SIZE" -gt 1 ]; then
-    EP_ARGS=(--enable-expert-parallel)
+    MODE_ARGS+=(
+        --enable-expert-parallel
+        --enable-ep-weight-filter
+        --moe-backend deep_gemm_amxf4_mega_moe
+    )
+fi
+if [ "$DP_ATTENTION" = "true" ]; then
+    MODE_ARGS+=(--prefill-schedule-interval 8)
+    if [ "$IS_DEP8" = "true" ]; then
+        # GPU-resident DEP8 gets a larger prefill token budget and chunks long
+        # prefills so decode latency stays bounded at high concurrency.
+        MODE_ARGS+=(
+            --max-num-batched-tokens 16384
+            --long-prefill-token-threshold 4096
+        )
+    else
+        MODE_ARGS+=(--max-num-batched-tokens 8192)
+    fi
 fi
 
-# AgentX concurrency counts live session trees, not individual requests.
-# Subagent fan-out can push instantaneous request concurrency above CONC, so
-# leave 2x headroom rather than clipping those bursts at the scheduler.
-MAX_NUM_SEQS=$((2 * CONC))
-if [ "$MAX_NUM_SEQS" -eq 128 ]; then
-    MAX_NUM_SEQS=136
+if [ "$DP_ATTENTION" = "true" ]; then
+    # The DEP source recipe enforces 2*CONC = DP_WORLD_SIZE*MAX_NUM_SEQS.
+    MAX_NUM_SEQS=$((2 * CONC / TP))
+else
+    # Preserve the previous TP4 scheduler headroom for agentic fan-out.
+    MAX_NUM_SEQS=$((2 * CONC))
 fi
+CUDA_GRAPH_CAPTURE_SIZES=""
+for ((capture_size = 1; capture_size <= MAX_NUM_SEQS; capture_size++)); do
+    if [ -n "$CUDA_GRAPH_CAPTURE_SIZES" ]; then
+        CUDA_GRAPH_CAPTURE_SIZES+=","
+    fi
+    CUDA_GRAPH_CAPTURE_SIZES+="$capture_size"
+done
+COMPILATION_CONFIG="{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[${CUDA_GRAPH_CAPTURE_SIZES}],\"mode\":0}"
 
 echo "Starting vllm server..."
 export TORCH_CUDA_ARCH_LIST="10.0"
 export PYTHONNOUSERSITE=1
 export VLLM_FLOAT32_MATMUL_PRECISION=high
 
-vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
---host 0.0.0.0 \
---port "$VLLM_BACKEND_PORT" \
---trust-remote-code \
---kv-cache-dtype fp8 \
---block-size 256 \
-"${PARALLEL_ARGS[@]}" \
-"${VLLM_CP_ARGS[@]}" \
-"${EP_ARGS[@]}" \
---compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \
---attention_config.use_fp4_indexer_cache=True \
---tokenizer-mode deepseek_v4 \
---tool-call-parser deepseek_v4 \
---enable-auto-tool-choice \
---reasoning-parser deepseek_v4 \
---enable-prefix-caching \
---no-disable-hybrid-kv-cache-manager \
---max-num-seqs "$MAX_NUM_SEQS" \
-"${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 &
+# DEP8 leaves more headroom for its larger prefill token budget.
+GPU_MEM_UTIL=0.96
+if [ "$IS_DEP8" = "true" ]; then
+    GPU_MEM_UTIL=0.92
+fi
+
+{ set +x; } 2>/dev/null
+VLLM_CMD=(
+    vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
+    --host 0.0.0.0
+    --port "$VLLM_BACKEND_PORT"
+    --gpu-memory-utilization "$GPU_MEM_UTIL"
+    --trust-remote-code
+    --no-enable-flashinfer-autotune
+    --no-disable-hybrid-kv-cache-manager
+    --max-num-seqs "$MAX_NUM_SEQS"
+    --kv-cache-dtype fp8
+    --block-size 256
+    --max-model-len 1048576
+    --attention-config '{"use_fp4_indexer_cache":true,"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}'
+    --disable-uvicorn-access-log
+    --tokenizer-mode deepseek_v4
+    --tool-call-parser deepseek_v4
+    --enable-auto-tool-choice
+    --reasoning-parser deepseek_v4
+    --compilation-config "$COMPILATION_CONFIG"
+    "${PARALLEL_ARGS[@]}"
+    "${TP_ARGS[@]}"
+    "${MODE_ARGS[@]}"
+    "${OFFLOAD_ARGS[@]}"
+)
+printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
+printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
+"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
 SERVER_PID=$!
 echo "Server PID: $SERVER_PID"
 
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
@@ -3576,7 +3576,7 @@ dsv4-fp4-b300-vllm:
       - { tp: 8, ep: 8, dp-attn: true, conc-start: 2048, conc-end: 2048 }
 
 dsv4-fp4-b300-vllm-agentic:
-  image: vllm/vllm-openai:v0.23.0
+  image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec
   model: deepseek-ai/DeepSeek-V4-Pro
   model-prefix: dsv4
   runner: cluster:b300-nv
@@ -3587,16 +3587,14 @@ dsv4-fp4-b300-vllm-agentic:
     agentic-coding:
     - dram-utilization: 0.80
       search-space:
-      # Compare native GPU-cache and MooncakeStore CPU-offload cliffs.
-      - { tp: 4, kv-offloading: none,  conc-list: [1, 2, 4, 6, 8, 16] }
-      - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [16, 18, 20, 24] }
-      # TP8 remains cache-resident through conc 52.
-      - { tp: 8, kv-offloading: none,  conc-list: [1, 2, 4, 6, 8, 16, 32, 40, 48, 52] }
-      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [52] }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none,  conc-list: [8, 16], router: { name: vllm-router, version: "0.1.14" } }
-      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [32], router: { name: vllm-router, version: "0.1.14" } }
-      # TP8 DEP retains representative low, peak, and transition points.
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none,  conc-list: [52, 72, 100, 128, 144], router: { name: vllm-router, version: "0.1.14" } }
+      # TP4 GPU-resident
+      - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32] }
+      # TP4 SimpleCPU
+      - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [28, 32, 36, 40] }
+      # DEP4 SimpleCPU
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
+      # DEP8 SimpleCPU
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066
diff --git a/perf-changelog.yaml b/perf-changelog.yaml
@@ -4901,3 +4901,11 @@
     - "Add EAGLE3 speculative-decoding arm for the Kimi K2.6 NVFP4 B300 AgentX recipe (draft lightseekorg/kimi-k2.6-eagle3.1-mla, TOKENSPEED_MLA attention backend with TRT-LLM ragged MLA kernel)."
     - "TP8/TP4 GPU-only KV points plus a TP4 native CPU-offload ladder via SimpleCPUOffloadConnector with lazy_offload off; TP4/DCP4 high-concurrency points (conc 32/64) using num_speculative_tokens=3 and synthetic_acceptance_length=2.88."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2228
+
+- config-keys:
+    - dsv4-fp4-b300-vllm-agentic
+  description:
+    - "Update B300 AgentX: KV offload, sparse DSV4 attention, mega-MoE, and FULL_DECODE_ONLY CUDA graphs."
+    - "Image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec"
+    - "B300: GPU-resident TP4/TP8 at conc [1,2,4,6,8,12,16,20,24,28,32] and DEP8 at conc [32,64,96,128,160,192,196,224,228] (max-num-batched-tokens 16384, long-prefill-token-threshold 4096, gpu-memory-utilization 0.92); TP4 SimpleCPU lazy-offload at conc [28,32,36,40]; DEP4 at conc [8,16,24,32,40,48,56,64,72] with both SimpleCPU and Mooncake 0.3.11.post1."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2241
