[LMCache] Add LMCache arm on tuned DSV4 FP4 B300 vLLM AgentX recipe / 在调优后的 DSV4 FP4 B300 vLLM AgentX 配方上新增 LMCache 分支

benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh

@@ -3,7 +3,8 @@ set -eo pipefail
 set -x
 
 # Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using vLLM.
-# v4pro-b300.yaml TP4, DEP4, and DEP8 recipe. SimpleCPUOffload / MooncakeStore
+# v4pro-b300.yaml TP4, DEP4, and DEP8 recipe. SimpleCPUOffload /
+# MooncakeStore / LMCache
 #
 # Image is configured in nvidia-master.yaml. The recipe uses FP8 KV cache,
 # sparse DeepSeek-V4 FlashInfer attention with an FP4 indexer cache, mega-MoE,
@@ -12,8 +13,9 @@ set -x
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# TP4, TP8, and DEP8 (TP8 + DP-attention) are GPU-resident (KV_OFFLOADING=none).
-# DEP4 uses KV_OFFLOADING=dram with KV_OFFLOAD_BACKEND=vllm-simple or mooncake.
+# GPU-resident arms (TP4 and DEP8) use KV_OFFLOADING=none. DRAM offload
+# arms (TP and DEP) use KV_OFFLOADING=dram with
+# KV_OFFLOAD_BACKEND=vllm-simple, mooncake, or lmcache.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -33,8 +35,8 @@ if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then
     exit 1
 fi
 
-# DEP8 (TP8 + DP-attention) is a GPU-resident, high-concurrency arm that is
-# tuned separately from the smaller DEP4 arm (larger prefill token budget,
+# DEP8 (TP8 + DP-attention) is a high-concurrency arm that is tuned
+# separately from the smaller DEP4 arm (larger prefill token budget,
 # long-prefill chunking, and a lower GPU-memory-utilization headroom).
 IS_DEP8=false
 if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then
@@ -89,11 +91,13 @@ export VLLM_USE_RUST_FRONTEND=1
 SERVER_LOG="$RESULT_DIR/server.log"
 ROUTER_LOG="$RESULT_DIR/router.log"
 MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
+LMCACHE_SERVER_LOG="$RESULT_DIR/lmcache_server.log"
 mkdir -p "$RESULT_DIR"
 
 SERVER_PID=""
 ROUTER_PID=""
 MOONCAKE_MASTER_PID=""
+LMCACHE_SERVER_PID=""
 
 # The generated TOTAL_CPU_DRAM_GB budget is proportional to allocated GPUs.
 # On cluster:b300-nv, dram-utilization=0.80 and DEP4 resolve to roughly the
@@ -186,6 +190,89 @@ EOF
         OFFLOAD_CONFIG='{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
         OFFLOAD_ARGS=(--kv-transfer-config "$OFFLOAD_CONFIG")
         ;;
+    lmcache)
+        require_agentic_kv_offload_backend lmcache
+        # The LMCache MP server owns the host-DRAM KV pool as one shared
+        # tier; vLLM ranks attach via LMCacheMPConnector, so the aggregate
+        # host budget is passed through undivided (unlike Mooncake's
+        # per-rank segments). Follows the LMCache DeepSeek-V4 recipe
+        # (docs.lmcache.ai/recipes/deepseek_v4_flash.html); LMCache handles
+        # DSV4's Sparse-MLA hybrid KV geometries automatically.
+        LMCACHE_VERSION=0.5.1
+        agentic_pip_install --quiet --no-cache-dir "lmcache==$LMCACHE_VERSION"
+        python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null
+
+        LMCACHE_HOST=127.0.0.1
+        LMCACHE_PORT=$((PORT + 12000))
+        LMCACHE_HTTP_PORT=$((PORT + 13000))
+        # LMCacheMPConnector concatenates lmcache.mp.host and port into the
+        # ZMQ endpoint. Bind the server to a raw host, but pass the connector
+        # a ZMQ-style host string.
+        LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST"
+        # Pool target derated to 75% of the aggregate budget: pinned host
+        # memory is unswappable and also consumes GPU-side mapping
+        # resources, so leave headroom for vLLM host buffers and the OS.
+        # Full-budget targets OOM-killed the node (host OOM-killer or
+        # cudaErrorMemoryAllocation) as the cache filled past ~2 TB during
+        # PR #2153 bring-up.
+        LMCACHE_L1_SIZE_GB=$((TOTAL_CPU_DRAM_GB * 3 / 4))
+        # The pool grows lazily from the initial allocation, so the full
+        # --l1-size-gb target is not pinned at startup.
+        LMCACHE_L1_INIT_SIZE_GB=20
+        LMCACHE_MQ_TIMEOUT=300
+        # Identical prefixes must hash to identical cache keys across DP ranks.
+        export PYTHONHASHSEED=0
+        # Per-engine scheduler stats every 5s, to diagnose per-DP-rank KV
+        # cache imbalance under the session-sticky router.
+        export VLLM_LOG_STATS_INTERVAL=5
+
+        echo "Starting LMCache MP server on port $LMCACHE_PORT..."
+        # One GPU-side transfer worker avoids concurrent-GPU-transfer stalls
+        # under heavy async-load pressure; CPU-side workers stay at 8.
+        lmcache server \
+            --host "$LMCACHE_HOST" \
+            --port "$LMCACHE_PORT" \
+            --http-host "$LMCACHE_HOST" \
+            --http-port "$LMCACHE_HTTP_PORT" \
+            --l1-size-gb "$LMCACHE_L1_SIZE_GB" \
+            --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" \
+            --max-gpu-workers 1 \
+            --max-cpu-workers 8 \
+            --chunk-size 1024 \
+            --l1-align-bytes 16384 \
+            --eviction-trigger-watermark 0.85 \
+            --eviction-ratio 0.10 \
+            --eviction-policy LRU \
+            --supported-transfer-mode lmcache_driven \
+            --no-separate-object-groups \
+            > "$LMCACHE_SERVER_LOG" 2>&1 &
+        LMCACHE_SERVER_PID=$!
+        LMCACHE_READY=0
+        for _ in $(seq 1 60); do
+            if ! kill -0 "$LMCACHE_SERVER_PID" 2>/dev/null; then
+                echo "LMCache server died during startup." >&2
+                cat "$LMCACHE_SERVER_LOG" >&2
+                exit 1
+            fi
+            if curl --output /dev/null --silent --fail \
+                "http://127.0.0.1:$LMCACHE_HTTP_PORT/healthcheck"; then
+                LMCACHE_READY=1
+                break
+            fi
+            sleep 2
+        done
+        if [ "$LMCACHE_READY" -ne 1 ]; then
+            echo "LMCache server did not become healthy in time." >&2
+            cat "$LMCACHE_SERVER_LOG" >&2
+            exit 1
+        fi
+
+        unset VLLM_USE_SIMPLE_KV_OFFLOAD
+        OFFLOAD_ARGS=(
+            --kv-transfer-config
+            "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":$LMCACHE_MQ_TIMEOUT}}"
+        )
+        ;;
     *)
         echo "Error: unsupported B300 KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2
         exit 1
@@ -199,7 +286,15 @@ fi
 
 TP_ARGS=()
 if [ "$DP_ATTENTION" = "true" ]; then
-    export PYTORCH_ALLOC_CONF=expandable_segments:True
+    # LMCacheMPConnector exports the KV cache to the LMCache server through
+    # legacy CUDA IPC handles, and expandable-segment (cuMem/VMM) allocations
+    # cannot be exported that way (register_kv_caches fails with
+    # cudaErrorInvalidValue, same failure mode as --enable-cumem-allocator on
+    # the B200 lmcache arm in PR #2231), so the lmcache arm keeps the stock
+    # caching allocator.
+    if [ "$KV_OFFLOAD_BACKEND" != "lmcache" ]; then
+        export PYTORCH_ALLOC_CONF=expandable_segments:True
+    fi
 else
     export VLLM_ALLREDUCE_USE_FLASHINFER=1
     export VLLM_FLASHINFER_ALLREDUCE_BACKEND=auto
@@ -217,8 +312,8 @@ fi
 if [ "$DP_ATTENTION" = "true" ]; then
     MODE_ARGS+=(--prefill-schedule-interval 8)
     if [ "$IS_DEP8" = "true" ]; then
-        # GPU-resident DEP8 gets a larger prefill token budget and chunks long
-        # prefills so decode latency stays bounded at high concurrency.
+        # DEP8 gets a larger prefill token budget and chunks long prefills
+        # so decode latency stays bounded at high concurrency.
         MODE_ARGS+=(
             --max-num-batched-tokens 16384
             --long-prefill-token-threshold 4096
@@ -254,6 +349,16 @@ GPU_MEM_UTIL=0.96
 if [ "$IS_DEP8" = "true" ]; then
     GPU_MEM_UTIL=0.92
 fi
+# The lmcache arm needs extra headroom on the other topologies too: the
+# LMCache MP server keeps a GPU worker (~0.8 GiB CUDA context + staging
+# buffers) on every GPU and the arm cannot run expandable segments (the KV
+# cache must stay legacy-CUDA-IPC-exportable), so at 0.96 the PR #2232
+# bring-up sweep hit DeepGEMM workspace/JIT-loader OOMs. 0.94 was validated
+# for lmcache TP and DEP4 in the same sweep; lmcache DEP8 keeps the 0.92
+# above (also validated).
+if [ "$KV_OFFLOAD_BACKEND" = "lmcache" ] && [ "$IS_DEP8" != "true" ]; then
+    GPU_MEM_UTIL=0.94
+fi
 
 { set +x; } 2>/dev/null
 VLLM_CMD=(

configs/nvidia-master.yaml

@@ -1753,6 +1753,17 @@ dsv4-fp4-b300-vllm-agentic:
       - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple, version: "904e4ec" }, conc-list: [32, 40, 48, 56, 64, 72], router: { name: vllm-router, version: "0.1.14" } }
       # DEP8 SimpleCPU
       - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } }
+      # LMCache arms: offset each official ladder above so the backends
+      # sample distinct points on the same recipe — TP4/DEP4 at +4 conc,
+      # DEP8 at +8 conc capped at 208. Bring-up validated in the PR #2232
+      # sweeps (runs 29463061871 and 29535333851): TP/DEP4 at
+      # gpu-memory-utilization 0.94, DEP8 at 0.92.
+      # TP4 LMCache (vs TP4 SimpleCPU)
+      - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [32, 36, 40, 44] }
+      # DEP4 LMCache (vs DEP4 SimpleCPU)
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [36, 44, 52, 60, 68, 76], router: { name: vllm-router, version: "0.1.14" } }
+      # DEP8 LMCache (vs GPU-resident DEP8)
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [72, 104, 120, 136, 152, 168, 184, 200], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b300-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066

perf-changelog.yaml

@@ -4982,3 +4982,14 @@
     - "DEP arm runs conc-list [48] only (was [48, 64, 96, 128, 192, 256, 512]): conc 48 is the frontier peak (163 tok/s/gpu out, ttft 2.8s); conc >=64 overflows the HBM+host radix cache (GPU hit 0.93->0.57 at 48->64) and thrashes on re-prefill, so it is strictly dominated"
     - "Low-latency TP8 arm (conc [1, 2, 4, 8, 16, 32], fp8 KV + cutedsl bf16 GEMM) unchanged; DEP mirrors the proven dsv4-fp4-b300-sglang tp8/ep8/dp-attn DEP arm"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2281
+
+- config-keys:
+    - dsv4-fp4-b300-vllm-agentic
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Add LMCache 0.5.1 DRAM KV-offload arms to the official B300 AgentX config from PR #2241 (sparse DSV4 FlashInfer attention, AMXF4 mega-MoE, FULL_DECODE_ONLY CUDA graphs, image vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec), so LMCache is benchmarked in the same sweep as the SimpleCPU and GPU-resident arms"
+    - "LMCache MP server (lmcache_driven transfer mode) + LMCacheMPConnector per PR #2153/#2231; L1 pool derated to 75% of TOTAL_CPU_DRAM_GB; PYTORCH_ALLOC_CONF=expandable_segments:True dropped on the lmcache arms only (VMM allocations cannot be CUDA-IPC-exported to the LMCache server, same failure mode as cuMem on B200); gpu-memory-utilization 0.92 on lmcache DEP8 (matches the official DEP8 derate) and 0.94 on lmcache TP4/DEP4 (at 0.96 the PR #2232 bring-up sweeps hit torch-pool, DeepGEMM-JIT, and cuBLAS-workspace OOMs; 23 lmcache points validated green in runs 29463061871/29535333851 after the derate)"
+    - "LMCache ladders offset the official arms: TP4 +4 conc [32, 36, 40, 44] vs SimpleCPU [28, 32, 36, 40]; DEP4 +4 conc [36, 44, 52, 60, 68, 76] vs SimpleCPU [32, 40, 48, 56, 64, 72]; DEP8 +8 conc capped at 208 [72, 104, 120, 136, 152, 168, 184, 200] vs GPU-resident [64, 96, 112, 128, 144, 160, 176, 192, 224]"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2232
+