diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh index cd5adec1b..5f0048e8c 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh @@ -3,7 +3,8 @@ set -eo pipefail set -x # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using vLLM. -# v4pro-b300.yaml TP4, DEP4, and DEP8 recipe. SimpleCPUOffload / MooncakeStore +# v4pro-b300.yaml TP4, DEP4, and DEP8 recipe. SimpleCPUOffload / +# MooncakeStore / LMCache # # Image is configured in nvidia-master.yaml. The recipe uses FP8 KV cache, # sparse DeepSeek-V4 FlashInfer attention with an FP4 indexer cache, mega-MoE, @@ -12,8 +13,9 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# TP4, TP8, and DEP8 (TP8 + DP-attention) are GPU-resident (KV_OFFLOADING=none). -# DEP4 uses KV_OFFLOADING=dram with KV_OFFLOAD_BACKEND=vllm-simple or mooncake. +# GPU-resident arms (TP4 and DEP8) use KV_OFFLOADING=none. DRAM offload +# arms (TP and DEP) use KV_OFFLOADING=dram with +# KV_OFFLOAD_BACKEND=vllm-simple, mooncake, or lmcache. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -33,8 +35,8 @@ if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then exit 1 fi -# DEP8 (TP8 + DP-attention) is a GPU-resident, high-concurrency arm that is -# tuned separately from the smaller DEP4 arm (larger prefill token budget, +# DEP8 (TP8 + DP-attention) is a high-concurrency arm that is tuned +# separately from the smaller DEP4 arm (larger prefill token budget, # long-prefill chunking, and a lower GPU-memory-utilization headroom). IS_DEP8=false if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then @@ -89,11 +91,13 @@ export VLLM_USE_RUST_FRONTEND=1 SERVER_LOG="$RESULT_DIR/server.log" ROUTER_LOG="$RESULT_DIR/router.log" MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" +LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" SERVER_PID="" ROUTER_PID="" MOONCAKE_MASTER_PID="" +LMCACHE_SERVER_PID="" # The generated TOTAL_CPU_DRAM_GB budget is proportional to allocated GPUs. # On cluster:b300-nv, dram-utilization=0.80 and DEP4 resolve to roughly the @@ -186,6 +190,89 @@ EOF OFFLOAD_CONFIG='{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' OFFLOAD_ARGS=(--kv-transfer-config "$OFFLOAD_CONFIG") ;; + lmcache) + require_agentic_kv_offload_backend lmcache + # The LMCache MP server owns the host-DRAM KV pool as one shared + # tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate + # host budget is passed through undivided (unlike Mooncake's + # per-rank segments). Follows the LMCache DeepSeek-V4 recipe + # (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles + # DSV4's Sparse-MLA hybrid KV geometries automatically. + LMCACHE_VERSION=0.5.1 + agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION" + python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null + + LMCACHE_HOST=127.0.0.1 + LMCACHE_PORT=$((PORT + 12000)) + LMCACHE_HTTP_PORT=$((PORT + 13000)) + # LMCacheMPConnector concatenates lmcache.mp.host and port into the + # ZMQ endpoint. Bind the server to a raw host, but pass the connector + # a ZMQ-style host string. + LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST" + # Pool target derated to 75% of the aggregate budget: pinned host + # memory is unswappable and also consumes GPU-side mapping + # resources, so leave headroom for vLLM host buffers and the OS. + # Full-budget targets OOM-killed the node (host OOM-killer or + # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during + # PR #2153 bring-up. + LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4)) + # The pool grows lazily from the initial allocation, so the full + # --l1-size-gb target is not pinned at startup. + LMCACHE_L1_INIT_SIZE_GB=20 + LMCACHE_MQ_TIMEOUT=300 + # Identical prefixes must hash to identical cache keys across DP ranks. + export PYTHONHASHSEED=0 + # Per-engine scheduler stats every 5s, to diagnose per-DP-rank KV + # cache imbalance under the session-sticky router. + export VLLM_LOG_STATS_INTERVAL=5 + + echo "Starting LMCache MP server on port $LMCACHE_PORT..." + # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls + # under heavy async-load pressure; CPU-side workers stay at 8. + lmcache server \ + --host "$LMCACHE_HOST" \ + --port "$LMCACHE_PORT" \ + --http-host "$LMCACHE_HOST" \ + --http-port "$LMCACHE_HTTP_PORT" \ + --l1-size-gb "$LMCACHE_L1_SIZE_GB" \ + --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \ + --max-gpu-workers 1 \ + --max-cpu-workers 8 \ + --chunk-size 1024 \ + --l1-align-bytes 16384 \ + --eviction-trigger-watermark 0.85 \ + --eviction-ratio 0.10 \ + --eviction-policy LRU \ + --supported-transfer-mode lmcache_driven \ + --no-separate-object-groups \ + > "$LMCACHE_SERVER_LOG" 2>&1 & + LMCACHE_SERVER_PID=$! + LMCACHE_READY=0 + for _ in $(seq 1 60); do + if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then + echo "LMCache server died during startup." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + if curl --output /dev/null --silent --fail \ + "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then + LMCACHE_READY=1 + break + fi + sleep 2 + done + if [ "$LMCACHE_READY" -ne 1 ]; then + echo "LMCache server did not become healthy in time." >&2 + cat "$LMCACHE_SERVER_LOG" >&2 + exit 1 + fi + + unset VLLM_USE_SIMPLE_KV_OFFLOAD + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}" + ) + ;; *) echo "Error: unsupported B300 KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2 exit 1 @@ -199,7 +286,15 @@ fi TP_ARGS=() if [ "$DP_ATTENTION" = "true" ]; then - export PYTORCH_ALLOC_CONF=expandable_segments:True + # LMCacheMPConnector exports the KV cache to the LMCache server through + # legacy CUDA IPC handles, and expandable-segment (cuMem/VMM) allocations + # cannot be exported that way (register_kv_caches fails with + # cudaErrorInvalidValue, same failure mode as --enable-cumem-allocator on + # the B200 lmcache arm in PR #2231), so the lmcache arm keeps the stock + # caching allocator. + if [ "$KV_OFFLOAD_BACKEND" != "lmcache" ]; then + export PYTORCH_ALLOC_CONF=expandable_segments:True + fi else export VLLM_ALLREDUCE_USE_FLASHINFER=1 export VLLM_FLASHINFER_ALLREDUCE_BACKEND=auto @@ -217,8 +312,8 @@ fi if [ "$DP_ATTENTION" = "true" ]; then MODE_ARGS+=(--prefill-schedule-interval 8) if [ "$IS_DEP8" = "true" ]; then - # GPU-resident DEP8 gets a larger prefill token budget and chunks long - # prefills so decode latency stays bounded at high concurrency. + # DEP8 gets a larger prefill token budget and chunks long prefills + # so decode latency stays bounded at high concurrency. MODE_ARGS+=( --max-num-batched-tokens 16384 --long-prefill-token-threshold 4096 @@ -254,6 +349,16 @@ GPU_MEM_UTIL=0.96 if [ "$IS_DEP8" = "true" ]; then GPU_MEM_UTIL=0.92 fi +# The lmcache arm needs extra headroom on the other topologies too: the +# LMCache MP server keeps a GPU worker (~0.8 GiB CUDA context + staging +# buffers) on every GPU and the arm cannot run expandable segments (the KV +# cache must stay legacy-CUDA-IPC-exportable), so at 0.96 the PR #2232 +# bring-up sweep hit DeepGEMM workspace/JIT-loader OOMs. 0.94 was validated +# for lmcache TP and DEP4 in the same sweep; lmcache DEP8 keeps the 0.92 +# above (also validated). +if [ "$KV_OFFLOAD_BACKEND" = "lmcache" ] && [ "$IS_DEP8" != "true" ]; then + GPU_MEM_UTIL=0.94 +fi { set +x; } 2>/dev/null VLLM_CMD=( diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 01ebad8b9..42abc8894 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1753,6 +1753,17 @@ dsv4-fp4-b300-vllm-agentic: - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } } # DEP8 SimpleCPU - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } } + # LMCache arms: offset each official ladder above so the backends + # sample distinct points on the same recipe — TP4/DEP4 at +4 conc, + # DEP8 at +8 conc capped at 208. Bring-up validated in the PR #2232 + # sweeps (runs 29463061871 and 29535333851): TP/DEP4 at + # gpu-memory-utilization 0.94, DEP8 at 0.92. + # TP4 LMCache (vs TP4 SimpleCPU) + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [32, 36, 40, 44] } + # DEP4 LMCache (vs DEP4 SimpleCPU) + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } } + # DEP8 LMCache (vs GPU-resident DEP8) + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [72, 104, 120, 136, 152, 168, 184, 200], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b300-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a76970e3b..8162941c3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4982,3 +4982,14 @@ - "DEP arm runs conc-list [48] only (was [48, 64, 96, 128, 192, 256, 512]): conc 48 is the frontier peak (163 tok/s/gpu out, ttft 2.8s); conc >=64 overflows the HBM+host radix cache (GPU hit 0.93->0.57 at 48->64) and thrashes on re-prefill, so it is strictly dominated" - "Low-latency TP8 arm (conc [1, 2, 4, 8, 16, 32], fp8 KV + cutedsl bf16 GEMM) unchanged; DEP mirrors the proven dsv4-fp4-b300-sglang tp8/ep8/dp-attn DEP arm" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2281 + +- config-keys: + - dsv4-fp4-b300-vllm-agentic + scenario-type: + - agentic-coding + description: + - "Add LMCache 0.5.1 DRAM KV-offload arms to the official B300 AgentX config from PR #2241 (sparse DSV4 FlashInfer attention, AMXF4 mega-MoE, FULL_DECODE_ONLY CUDA graphs, image vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec), so LMCache is benchmarked in the same sweep as the SimpleCPU and GPU-resident arms" + - "LMCache MP server (lmcache_driven transfer mode) + LMCacheMPConnector per PR #2153/#2231; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB; PYTORCH_ALLOC_CONF=expandable_segments:True dropped on the lmcache arms only (VMM allocations cannot be CUDA-IPC-exported to the LMCache server, same failure mode as cuMem on B200); gpu-memory-utilization 0.92 on lmcache DEP8 (matches the official DEP8 derate) and 0.94 on lmcache TP4/DEP4 (at 0.96 the PR #2232 bring-up sweeps hit torch-pool, DeepGEMM-JIT, and cuBLAS-workspace OOMs; 23 lmcache points validated green in runs 29463061871/29535333851 after the derate)" + - "LMCache ladders offset the official arms: TP4 +4 conc [32, 36, 40, 44] vs SimpleCPU [28, 32, 36, 40]; DEP4 +4 conc [36, 44, 52, 60, 68, 76] vs SimpleCPU [32, 40, 48, 56, 64, 72]; DEP8 +8 conc capped at 208 [72, 104, 120, 136, 152, 168, 184, 200] vs GPU-resident [64, 96, 112, 128, 144, 160, 176, 192, 224]" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2232 +