diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh new file mode 100755 index 0000000000..85ffd64609 --- /dev/null +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh @@ -0,0 +1,315 @@ +#!/usr/bin/env bash +set -eo pipefail +set -x + +# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B200 using vLLM, +# with MTP speculative decoding (num_speculative_tokens=3): synthetic acceptance +# length 2.49 for throughput, real target verification for the EVAL_ONLY eval. +# +# Identical to dsv4_fp4_b200_vllm.sh (same image, engine args, offload, GPU +# topologies, and agentic aiperf rig) with exactly two MTP deltas: +# --speculative-config: synthetic acceptance length 2.49 (throughput) vs real MTP (EVAL_ONLY); see the SPEC_CONFIG block +# --max-cudagraph-capture-size expressed in TOKENS (see the capture block below). +# +# Mirrors the fixed-seq-len parallelism options (pure TP and DEP) so the +# agentic sweep can probe both interactivity and throughput regimes: +# pure TP (DP_ATTENTION=false, EP_SIZE=1): attention TP-sharded across +# all $TP GPUs in a single engine. Lower TPOT, lower batch. +# TP+EP (DP_ATTENTION=false, EP_SIZE>1): attention TP-sharded, MoE +# experts EP-sharded within the TP group. +# DEP (DP_ATTENTION=true, EP_SIZE>1): per-DP-rank attention with +# experts EP-sharded across DP ranks (per the vLLM blog recipe). +# Highest aggregate throughput at large CONC. +# +# Image is configured in nvidia-master.yaml. block_size=256, +# kv-cache-dtype=fp8, FLASHINFER_MLA_SPARSE_DSV4 attention with the FP4 indexer +# cache, FULL_DECODE_ONLY cudagraph capture, and (in EP tiers) mega-MoE backend. +# +# Required env vars: +# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR +# +# Pure TP is GPU-resident (KV_OFFLOADING=none). DEP tiers offload KV to host +# DRAM: KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=vllm-simple or mooncake. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +if [ -z "$DCP_SIZE" ]; then + DCP_SIZE=1 +fi +if [ -z "$PCP_SIZE" ]; then + PCP_SIZE=1 +fi +VLLM_CP_ARGS=() +if [ "$DCP_SIZE" -gt 1 ]; then + VLLM_CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE") +fi +if [ "$PCP_SIZE" -gt 1 ]; then + VLLM_CP_ARGS+=(--prefill-context-parallel-size "$PCP_SIZE") +fi + +GPU_COUNT=$((TP * PCP_SIZE)) +if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2 + exit 1 +fi +export GPU_COUNT + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "$MODEL_PATH" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +nvidia-smi + +# ---- Resolve traces and install deps ---------------------------------------- +resolve_trace_source +install_agentic_deps + +# vllm-project/router expands the one HTTP backend into one logical worker per +# DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's +# X-Correlation-ID is stable for every turn of a conversation; alias it to the +# router's preferred X-Session-ID header. +USE_VLLM_ROUTER=false +VLLM_BACKEND_PORT="$PORT" +if [ "$DP_ATTENTION" = "true" ]; then + USE_VLLM_ROUTER=true + VLLM_BACKEND_PORT=$((PORT + 1)) + VLLM_ROUTER_VERSION=0.1.14 + VLLM_ROUTER_POLICY=consistent_hash + VLLM_ROUTER_METRICS_PORT=$((PORT + 10000)) + export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 + agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION" +fi + +# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s. +export VLLM_ENGINE_READY_TIMEOUT_S=3600 + +# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while +# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's +# store mask. 32k matches the trace-replay tuning validated for this workload. +export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 +export VLLM_USE_V2_MODEL_RUNNER=1 +export VLLM_USE_RUST_FRONTEND=1 +export VLLM_DSV4_MEGA_FP8_COMBINE=1 +export VLLM_RPC_TIMEOUT=600000 + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +ROUTER_LOG="$RESULT_DIR/router.log" +MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +ROUTER_PID="" +MOONCAKE_MASTER_PID="" + +OFFLOAD_ARGS=() +case "$KV_OFFLOAD_BACKEND" in + "") + require_agentic_kv_offload_none + ;; + vllm-simple) + require_agentic_kv_offload_backend vllm-simple + CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT )) + # Identical prefixes must hash to identical block keys across DP ranks. + export PYTHONHASHSEED=42 + OFFLOAD_CONFIG=$(cat </dev/null + + MOONCAKE_MASTER_PORT=$((PORT + 12000)) + MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" + cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & + MOONCAKE_MASTER_PID=$! + sleep 2 + if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then + echo "Mooncake master died during startup." >&2 + cat "$MOONCAKE_MASTER_LOG" >&2 + exit 1 + fi + unset VLLM_USE_SIMPLE_KV_OFFLOAD + OFFLOAD_ARGS=( + --kv-transfer-config + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + ) + ;; + *) + echo "Error: unsupported B200 KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2 + exit 1 + ;; +esac + +PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) +if [ "$DP_ATTENTION" = "true" ]; then + PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") +fi + +EP_ARGS=() +FAST_MOE_ARGS=() +if [ "$EP_SIZE" -gt 1 ]; then + EP_ARGS=(--enable-expert-parallel) + FAST_MOE_ARGS=( + --moe-backend deep_gemm_amxf4_mega_moe + --enable-ep-weight-filter + --prefill-schedule-interval 16 + ) +fi + +# AgentX concurrency counts live session trees, not individual requests. +# Subagent fan-out can push instantaneous request concurrency above CONC, so +# leave 2x headroom rather than clipping those bursts at the scheduler. +MAX_NUM_SEQS=$((2 * CONC)) + +# MTP: cudagraph capture sizes are in TOKENS. A uniform decode batch of S seqs +# verifies S*(1+num_speculative_tokens) tokens, so cap capture at +# MAX_NUM_SEQS*(1+N) tokens -- otherwise vLLM's FULL_DECODE_ONLY ladder tops out +# at MAX_NUM_SEQS/(1+N) seqs and the largest decode batches fall back to eager. +NUM_SPEC_TOKENS=3 +# Throughput pins synthetic MTP acceptance to the dsv4-pro golden AL (thinking_on, +# num_speculative_tokens=3, golden_al_distribution/dsv4_mtp.yaml). The EVAL_ONLY +# accuracy run uses real target verification instead -- synthetic acceptance +# bypasses verification and corrupts the SWE-bench eval (0.0000 score). +SYNTHETIC_ACCEPT_LEN=2.49 +if [ "${EVAL_ONLY:-false}" = "true" ]; then + SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" +else + SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" +fi +TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS)) +MAX_CUDAGRAPH_CAPTURE_SIZE=$((MAX_NUM_SEQS * TOKENS_PER_SEQ)) + +echo "Starting vllm server..." +export TORCH_CUDA_ARCH_LIST="10.0" +export PYTHONNOUSERSITE=1 +export VLLM_FLOAT32_MATMUL_PRECISION=high + +{ set +x; } 2>/dev/null +VLLM_CMD=( + vllm serve "$MODEL_PATH" --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$VLLM_BACKEND_PORT" + --trust-remote-code + --kv-cache-dtype fp8 + --block-size 256 + --max-model-len 1048576 + --gpu-memory-utilization 0.9 + --numa-bind + --enable-cumem-allocator + --no-enable-flashinfer-autotune + --tokenizer-mode deepseek_v4 + --tool-call-parser deepseek_v4 + --enable-auto-tool-choice + --reasoning-parser deepseek_v4 + --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + --speculative-config "$SPEC_CONFIG" + --no-disable-hybrid-kv-cache-manager + --disable-uvicorn-access-log + --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + --max-num-seqs "$MAX_NUM_SEQS" + --max-cudagraph-capture-size "$MAX_CUDAGRAPH_CAPTURE_SIZE" + "${PARALLEL_ARGS[@]}" + "${VLLM_CP_ARGS[@]}" + "${EP_ARGS[@]}" + "${FAST_MOE_ARGS[@]}" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "$USE_VLLM_ROUTER" = "true" ]; then + echo "Starting native vLLM router on port $PORT for $TP DP ranks..." + vllm-router \ + --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ + --policy "$VLLM_ROUTER_POLICY" \ + --intra-node-data-parallel-size "$TP" \ + --host 0.0.0.0 \ + --port "$PORT" \ + --prometheus-host 127.0.0.1 \ + --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \ + --request-timeout-secs 14400 \ + --disable-retries > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + echo "Router PID: $ROUTER_PID" + wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" +fi + +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 3aac1b8378..a6d9a5872d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -964,6 +964,27 @@ dsv4-fp4-b200-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 68, 72, 80], router: { name: vllm-router, version: "0.1.14" } } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } } +dsv4-fp4-b200-vllm-agentic-mtp: + image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b200-dgxc + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + # Pure TP GPU-resident + MTP (num_speculative_tokens=3) + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8] } + # Pure TP SimpleCPU + MTP (num_speculative_tokens=3) + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [8, 12, 16] } + # DEP SimpleCPU + MTP (num_speculative_tokens=3) + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [8, 16, 24, 32, 40, 48, 56, 64, 68, 72, 80], router: { name: vllm-router, version: "0.1.14" } } + # DEP Mooncake + MTP (num_speculative_tokens=3) + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, spec-decoding: mtp, conc-list: [12, 20, 28, 36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } } + dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 892e3ee2b6..2f989bfd15 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4974,3 +4974,11 @@ - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity" - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-mtp + description: + - "Add MTP speculative-decoding for the B200 vLLM AgentX recipe as a separate config-key, so the existing dsv4-fp4-b200-vllm-agentic aggregate is not re-run." + - "dsv4-fp4-b200-vllm-agentic-mtp: MTP twins (--speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":3}) of the aggregate arms -- TP8 GPU-resident, TP8 SimpleCPU, DEP8 SimpleCPU, DEP8 Mooncake, each mirroring its non-MTP conc-list; routed via spec-decoding=mtp to dsv4_fp4_b200_vllm_mtp.sh." + - "MTP capture sizes are in TOKENS: --max-cudagraph-capture-size scaled to MAX_NUM_SEQS*(1+num_speculative_tokens) (= MAX_NUM_SEQS*4) so FULL_DECODE_ONLY covers the largest decode batches." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2259