diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh index e5dde8d120..f43c81e1c6 100755 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh @@ -83,8 +83,20 @@ if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; t export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}" # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}" - # HiCache ratio (host pool = ratio * GPU KV pool). Default derived in server_sglang.sh. - export HICACHE_RATIO="${HICACHE_RATIO:-}" + # HiCache ratio (host pool = ratio * GPU KV pool). + export HICACHE_RATIO="${HICACHE_RATIO:-4}" + # server_sglang.sh prefers an absolute --hicache-size (derived from + # TOTAL_CPU_DRAM_GB, the sweep generator's per-node DRAM budget) over + # --hicache-ratio whenever TOTAL_CPU_DRAM_GB is set. DSv4 wants the + # ratio-based pool instead. Use FORCE_HICACHE_RATIO to opt out of the + # --hicache-size path rather than unsetting TOTAL_CPU_DRAM_GB itself: + # that var is also the shared client-side gate (benchmark_lib.sh requires + # it to be a positive integer whenever KV_OFFLOADING=dram) and gets + # forwarded into the aiperf sibling container's client.env, so unsetting + # it here made the client fail its own env validation before benchmarking + # ("DRAM KV offloading requires a positive configured TOTAL_CPU_DRAM_GB + # capacity") even though the servers came up fine. + export FORCE_HICACHE_RATIO=1 # ── HiCache layout/backend by tier ── # L3 (Mooncake): page_first + direct + write_through + storage=mooncake @@ -122,7 +134,7 @@ export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}" export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-32768}" # ── SGLang PD router policy + server metrics ── -export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-cache_aware}" +export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-consistent_hashing}" export ENABLE_METRICS="${ENABLE_METRICS:-1}" # ── MTP ── diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index baa917b449..a4c557d860 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -275,6 +275,26 @@ else # FIXME: WA for latest upstream 0305 image export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH} + # Decode CUDA-graph capture crash on ROCm 7.2.0 (TP8+EP8, mori a2a). + # Symptom: during decode cuda-graph capture, the torch ProcessGroupNCCL + # *watchdog* thread calls hipEventQuery() to poll in-flight NCCL work. + # ROCm <= 7.2.0's HIP runtime does NOT honor cudaStreamCaptureModeThreadLocal, + # so the watchdog's cross-thread query touches the main thread's active + # capture and invalidates it -> "HIP error: operation not permitted on an + # event last recorded in a capturing stream (hipErrorCapturedEvent)" -> + # watchdog aborts -> "Rank 0 scheduler died during initialization" (-6). + # This is a HIP runtime bug, not OOM and not a mori/deepep-mode bug (it + # fires for --deepep-mode normal and auto alike; EP8+mori just adds NCCL + # PGs that make the watchdog race fire). Refs: sgl-project/sglang#29235, + # #24011; ROCm/hip#3876; pytorch/pytorch#176251. + # Real fix = ROCm 7.2.2+ (honors THREAD_LOCAL). Until the base image is + # bumped, TORCH_NCCL_BLOCKING_WAIT=true makes NCCL work completion use a + # blocking wait instead of the async watchdog hipEventQuery poll, so no + # event is queried during capture. CUDA graph stays fully enabled. + export TORCH_NCCL_BLOCKING_WAIT="${TORCH_NCCL_BLOCKING_WAIT:-1}" + export NCCL_BLOCKING_WAIT="${NCCL_BLOCKING_WAIT:-1}" + # export NCCL_DEBUG="${NCCL_DEBUG:-INFO}" + # ========================================================================= # DeepSeek-V4-Pro PD recipe overrides # Placed at the end of the SGLang env block so it wins over the global diff --git a/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh b/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh new file mode 100755 index 0000000000..9e3df3b3be --- /dev/null +++ b/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# GPU sanity gate, run on each node AFTER all containers are stopped. +# +# At that point VRAM should fall back to driver baseline. Any large remaining +# usage means a bare (non-containerized) process is hogging the GPU -- which +# would otherwise surface as an opaque "memory capacity is unbalanced" OOM in +# the middle of model load ~30 min into the job. We fail fast and name the +# offender instead. We only report; we never kill another user's process. +# +# Ported from https://github.com/ROCm/InferenceY/blob/main/benchmarks/multi_node/amd_utils/gpu_sanity.sh +set -uo pipefail + +# Fail the node if any single GPU still has >= this many GB used. +GPU_BUSY_GB="${GPU_BUSY_GB:-8}" +# Poll a few times to give docker stop time to release VRAM before judging. +GPU_DRAIN_TRIES="${GPU_DRAIN_TRIES:-6}" + +if ! command -v rocm-smi >/dev/null 2>&1; then + echo "[gpu-sanity] rocm-smi not found on $(hostname); skipping GPU check" >&2 + exit 0 +fi + +busy_gb=0 +for _ in $(seq "$GPU_DRAIN_TRIES"); do + used=$(rocm-smi --showmeminfo vram --json 2>/dev/null \ + | grep -oE '"VRAM Total Used Memory \(B\)": "[0-9]+"' \ + | grep -oE '[0-9]+' | sort -n | tail -1) + busy_gb=$(( ${used:-0} / 1024 / 1024 / 1024 )) + if [ "$busy_gb" -lt "$GPU_BUSY_GB" ]; then + exit 0 + fi + sleep 5 +done + +echo "FATAL: $(hostname) GPU still has ${busy_gb}GB used after stopping all containers" >&2 +echo " (threshold ${GPU_BUSY_GB}GB) -- a bare (non-containerized) process is hogging the GPU:" >&2 +rocm-smi --showpids >&2 || true +exit 1 diff --git a/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh b/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh new file mode 100755 index 0000000000..86acabbab0 --- /dev/null +++ b/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# Pre-flight RDMA QoS/DCQCN validation for a single node, run by job.slurm via +# `srun` across every allocated node BEFORE any container/GPU time is spent. +# +# A misconfigured NIC (PFC not covering the RoCE priority, DCQCN disabled, ...) +# doesn't make MoRI's cross-node EP/RDMA transfers error out -- it just quietly +# degrades, and the only symptom is unexplained tail latency or throughput +# variance in the benchmark numbers hours later. Failing fast here, before the +# job burns node-hours, is much cheaper than debugging that after the fact. +# +# check_qos()/check_dcqcn() below are trimmed/adapted from ROCm/mori's +# tools/env_check.sh: +# https://github.com/ROCm/mori/blob/main/tools/env_check.sh +# The upstream script's expensive ib_write_bw/ib_write_lat full-mesh bandwidth +# and latency tests are intentionally NOT ported here -- this runs before +# every single job, so it must be fast (seconds), not a multi-minute +# fabric-wide benchmark of its own. +# +# Scoped to AMD Pollara (ionic) NICs via nicctl, matching this repo's current +# fleet (see env.sh's nicctl-based MORI_RDMA_TC/SL detection). If bnxt_re or +# mlx5 NICs are added to the fleet, port the bnxt_*/mlx5_* check_* functions +# from the upstream script following the same pattern. +# +# Exit code: 0 = OK (or gracefully skipped, e.g. no ionic NICs on this host), +# 1 = hard QoS/DCQCN misconfiguration -- do not proceed with the run. +set -uo pipefail + +AINIC_MIN_VER="1.117.5-a-45" # minimum recommended AINIC firmware for IBGDA + +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[0;33m' +NC='\033[0m' + +log_ok() { echo -e "[$(hostname -s)] ${GREEN}[OK]${NC} $*"; } +log_fail() { echo -e "[$(hostname -s)] ${RED}[FAIL]${NC} $*"; } +log_warn() { echo -e "[$(hostname -s)] ${YELLOW}[WARN]${NC} $*"; } +die() { log_fail "$@"; exit 1; } + +# version_ge -> true if >= (dotted/hyphenated, via sort -V) +version_ge() { + local cand="$1" min="$2" + [[ "$cand" == "$min" ]] && return 0 + [[ "$(printf '%s\n%s\n' "$cand" "$min" | sort -V | head -1)" == "$min" ]] +} + +# check_ainic_version_recommendation +# warns if firmware is on the IBGDA-incapable 1.117.1 branch, or below the +# recommended minimum for cross-node MORI (EP over RDMA / IBGDA). +check_ainic_version_recommendation() { + local ver="$1" + [[ -n "$ver" ]] || { log_warn "cannot verify AINIC firmware version against recommendation (empty)"; return; } + if [[ "$ver" =~ ^1\.117\.1([.-]|$) ]]; then + log_warn "AINIC firmware $ver is on the 1.117.1 branch, which does NOT support IBGDA -- upgrade to >= $AINIC_MIN_VER" + elif version_ge "$ver" "$AINIC_MIN_VER"; then + log_ok "AINIC firmware $ver meets the recommended minimum (>= $AINIC_MIN_VER) for cross-node IBGDA" + else + log_warn "AINIC firmware $ver is below the recommended minimum (>= $AINIC_MIN_VER) for cross-node IBGDA" + fi +} + +# check_versions() -- informational only (never hard-fails the job). +check_versions() { + local fw_output sw_output + fw_output=$(sudo nicctl show version firmware 2>/dev/null) + sw_output=$(sudo nicctl show version host-software 2>/dev/null) + + local fw_versions fw_count + fw_versions=$(echo "$fw_output" | grep -i "firmware" | awk '{print $NF}' | sort -u) + fw_count=$(echo "$fw_versions" | grep -c . || true) + if [[ "$fw_count" -ne 1 ]]; then + log_warn "firmware versions not consistent across NICs:" + echo "$fw_versions" + local v + while read -r v; do [[ -n "$v" ]] && check_ainic_version_recommendation "$v"; done <<< "$fw_versions" + else + log_ok "firmware : $fw_versions" + check_ainic_version_recommendation "$fw_versions" + fi + + local nicctl_ver + nicctl_ver=$(echo "$sw_output" | grep "nicctl" | awk '{print $NF}') + [[ -n "$nicctl_ver" ]] && log_ok "nicctl : $nicctl_ver" || log_warn "cannot determine nicctl version" +} + +# check_qos() -- HARD gate: classification type must be DSCP, and PFC no-drop +# must be enabled and cover every no-drop priority. Dies (exit 1) otherwise. +check_qos() { + local qos_output + qos_output=$(sudo nicctl show qos 2>/dev/null) + [[ -n "$qos_output" ]] || die "sudo nicctl show qos returned nothing" + + local class_type + class_type=$(echo "$qos_output" | grep "Classification type" | head -1 | awk '{print $NF}') + [[ "$class_type" == "DSCP" ]] || die "classification type is '$class_type', expected 'DSCP'" + log_ok "classification type : DSCP" + + local nd_prio_raw + nd_prio_raw=$(echo "$qos_output" | grep "PFC no-drop priorities" | head -1 | awk '{print $NF}') + [[ -n "$nd_prio_raw" ]] || die "cannot find PFC no-drop priority" + local nd_prios=() + IFS=',' read -ra nd_prios <<< "$nd_prio_raw" + log_ok "no-drop priorities : ${nd_prios[*]}" + + local pfc_bitmap + pfc_bitmap=$(echo "$qos_output" | grep "PFC priority bitmap" | head -1 | awk '{print $NF}') + [[ -n "$pfc_bitmap" && "$pfc_bitmap" != "0x0" ]] || die "PFC is not enabled (bitmap=$pfc_bitmap)" + local p + for p in "${nd_prios[@]}"; do + (( pfc_bitmap & (1 << p) )) || die "PFC bitmap $pfc_bitmap does not cover priority $p" + done + log_ok "PFC enabled for priorities ${nd_prios[*]} (bitmap=$pfc_bitmap)" +} + +# check_dcqcn() -- HARD gate: DCQCN must be enabled on every ROCE device, and +# the CNP DSCP must be consistent across NICs. Exits 1 otherwise. +check_dcqcn() { + local dcqcn_output + dcqcn_output=$(sudo nicctl show dcqcn 2>/dev/null) + [[ -n "$dcqcn_output" ]] || die "sudo nicctl show dcqcn returned nothing" + + local total + total=$(echo "$dcqcn_output" | grep -c "ROCE device") + [[ "$total" -gt 0 ]] || die "no ROCE devices found in dcqcn output" + + local disabled + disabled=$(echo "$dcqcn_output" | grep "Status" | grep -v "Enabled" || true) + if [[ -n "$disabled" ]]; then + log_fail "some ROCE devices have DCQCN disabled:" + echo "$disabled" + exit 1 + fi + log_ok "DCQCN enabled on all $total ROCE devices" + + local cnp_values cnp_count + cnp_values=$(echo "$dcqcn_output" | grep "DSCP value used for CNP" | awk '{print $NF}' | sort -u) + cnp_count=$(echo "$cnp_values" | grep -c . || true) + [[ "$cnp_count" -eq 1 ]] || die "CNP DSCP not consistent across NICs: $cnp_values" + log_ok "CNP DSCP = $cnp_values (consistent across all NICs)" +} + +# ============================= main ============================= + +if ! command -v nicctl &>/dev/null; then + log_warn "nicctl not found on $(hostname -s) -- skipping RDMA QoS/DCQCN pre-flight check (not an ionic NIC host, or nicctl not on PATH)" + exit 0 +fi + +# nicctl exits 0 even with no NIC present, so check its output rather than its exit code. +_nicctl_probe=$(sudo nicctl show version firmware 2>&1 || true) +if echo "$_nicctl_probe" | grep -qiE 'No AMD NICs|Invalid card handle|Failed to get NIC'; then + log_warn "nicctl present but no ionic NIC detected/accessible on $(hostname -s) -- skipping RDMA QoS/DCQCN pre-flight check" + exit 0 +fi + +check_versions +check_qos +check_dcqcn + +log_ok "RDMA QoS/DCQCN pre-flight check passed on $(hostname -s)" diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 863d9d7635..3c86f6fa06 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -345,6 +345,29 @@ export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) +# ============================================================================= +# RDMA QoS / DCQCN Pre-flight Check +# ============================================================================= +# Gate the run on NIC QoS (PFC/DSCP) and DCQCN config on every node before +# any container/GPU time is spent. Runs on the bare host (nicctl is a host +# tool). See rdma_check.sh for details. +RDMA_CHECK_SCRIPT="$(pwd)/helpers/rdma_check.sh" +if [[ "${SKIP_RDMA_CHECK:-0}" == "1" ]]; then + echo "[INFO] SKIP_RDMA_CHECK=1 set; skipping RDMA QoS/DCQCN pre-flight check" +elif [[ -f "$RDMA_CHECK_SCRIPT" ]]; then + echo "Checking RDMA QoS/DCQCN configuration on all $NUM_NODES allocated node(s)..." + srun --nodelist="$SELECTED_NODELIST_SRUN" --ntasks=$NUM_NODES bash "$RDMA_CHECK_SCRIPT" + RDMA_CHECK_RC=$? + if [[ $RDMA_CHECK_RC -ne 0 ]]; then + echo "FATAL: RDMA QoS/DCQCN pre-flight check failed on one or more nodes (see [FAIL] lines above)." + echo " Set SKIP_RDMA_CHECK=1 to bypass (not recommended -- MoRI cross-node transfers would run unprotected by PFC/DCQCN)." + exit 1 + fi + echo "RDMA QoS/DCQCN pre-flight check passed on all $NUM_NODES node(s)" +else + echo "[WARN] $RDMA_CHECK_SCRIPT not found; skipping RDMA QoS/DCQCN pre-flight check" +fi + cleanup() { echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning up container + stale logs..." # Backstop: on scancel/timeout/step-hang the foreground `exec docker run` @@ -494,6 +517,7 @@ fi HICACHE_MC_CONFIG="${BENCHMARK_LOGS_DIR}/hicache_mc_${SLURM_JOB_ID}.env" cat > "$HICACHE_MC_CONFIG" </dev/null || true diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index 3d6d7b8c19..0ae652e275 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -365,19 +365,19 @@ DeepSeek-R1-0528-MXFP4-v2: DeepSeek-V4-Pro-AgentX: base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error" - dp_flags: "--enable-dp-attention --enable-prefill-delayer --enable-two-batch-overlap" + dp_flags: "--enable-dp-attention --enable-prefill-delayer" ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" prefill: - mem_fraction_static: 0.8 + mem_fraction_static: 0.72 disable_radix_cache: false - disable_cuda_graph: false + disable_cuda_graph: true dp: max_running_requests: 1024 chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" # dsv4 compressor kernel uint16 token cap (255*256) context_length: 1048576 # max_total_tokens: 1048576 no_dp: - max_running_requests: 128 + max_running_requests: 64 # Small prefill chunks interleave long-context agentic prefills across # requests instead of letting one ~100K-token prefill monopolize the # engine (the conc>=16 queue-saturation / decode-stall failure mode). diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index f801bfa837..746b92a313 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -530,6 +530,12 @@ if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then # per-node DRAM budget computed by the sweep generator (enforcement); fall # back to --hicache-ratio (relative to the GPU KV pool) when no budget is # provided, keeping configs that predate the budget unchanged. + # FORCE_HICACHE_RATIO lets a recipe opt into ratio-based sizing without + # unsetting TOTAL_CPU_DRAM_GB — that var is also the shared client-side + # gate (benchmark_lib.sh requires it whenever KV_OFFLOADING=dram) and is + # forwarded verbatim into client.env below, so unsetting it here would + # make the aiperf client container fail its own env validation before + # ever sending a request. HICACHE_RATIO="${HICACHE_RATIO:-5}" HICACHE_SIZING_FLAGS="--hicache-ratio ${HICACHE_RATIO}" # DeepSeek V4's hybrid HiCache pool rejects --hicache-size (requires @@ -537,7 +543,9 @@ if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then # See sglang _deepseek_v4_num_host_pages() (raises ValueError when # server_args.hicache_size > 0): # https://github.com/sgl-project/sglang/blob/9dd57ef8c48e2cd82292d849f01e2130c5203e67/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py#L262-L266 - if [[ -n "${TOTAL_CPU_DRAM_GB:-}" && "${TOTAL_CPU_DRAM_GB}" -gt 0 && "${MODEL_NAME}" != *DeepSeek-V4* ]]; then + # FORCE_HICACHE_RATIO additionally lets a recipe opt into ratio-based sizing + # for any other model without unsetting TOTAL_CPU_DRAM_GB (see comment above). + if [[ "${FORCE_HICACHE_RATIO:-0}" != "1" && -n "${TOTAL_CPU_DRAM_GB:-}" && "${TOTAL_CPU_DRAM_GB}" -gt 0 && "${MODEL_NAME}" != *DeepSeek-V4* ]]; then # TOTAL_CPU_DRAM_GB is the prefill worker's per-node budget (only prefill # offloads KV to CPU DRAM today); --hicache-size is per rank per host # pool. A prefill server may span nodes (PREFILL_TP_SIZE is its total @@ -776,12 +784,17 @@ if [ "$NODE_RANK" -eq 0 ]; then # across the agentic trace; round_robin decode keeps the single decode worker # fed evenly. Override via ROUTER_RESILIENCE_FLAGS / ROUTER_POLICY_FLAGS. ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:---disable-circuit-breaker --health-failure-threshold 100 --health-check-timeout-secs 600 --health-check-interval-secs 30}" - ROUTER_PREFILL_POLICY="${ROUTER_PREFILL_POLICY:-cache_aware}" - ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" + # server_sglang.sh previously read ROUTER_PREFILL_POLICY, but the recipe + # scripts export PREFILL_ROUTER_POLICY, so the recipe's policy override was + # silently ignored and the router always fell back to this hardcoded + # default. Also comment out ROUTER_DECODE_POLICY for now (superseded by + # --dp-aware below). + ROUTER_PREFILL_POLICY="${PREFILL_ROUTER_POLICY:-consistent_hashing}" + # ROUTER_DECODE_POLICY="${ROUTER_DECODE_POLICY:-round_robin}" ROUTER_CACHE_THRESHOLD="${ROUTER_CACHE_THRESHOLD:-0.3}" ROUTER_BALANCE_ABS_THRESHOLD="${ROUTER_BALANCE_ABS_THRESHOLD:-2}" ROUTER_BALANCE_REL_THRESHOLD="${ROUTER_BALANCE_REL_THRESHOLD:-1.1}" - ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --prefill-policy ${ROUTER_PREFILL_POLICY} --decode-policy ${ROUTER_DECODE_POLICY} --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" + ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --dp-aware --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" else # DI router config (8k1k branch, run 28696443568): with defaults the per-worker # circuit stays OPEN for cb-timeout-duration-secs=60 before a half-open retrial. @@ -975,8 +988,8 @@ if [ "$NODE_RANK" -eq 0 ]; then ENABLE_METRICS IS_AGENTIC CLEAR_CACHE_BETWEEN_CONC \ DISAGG IS_MULTINODE \ TP EP_SIZE DP_ATTENTION DCP_SIZE PCP_SIZE \ - PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_DP_ATTN PREFILL_HARDWARE \ - DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_HARDWARE \ + PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_DP_ATTN PREFILL_ENABLE_DP PREFILL_HARDWARE \ + DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_ENABLE_DP DECODE_HARDWARE \ KV_OFFLOADING KV_OFFLOAD_BACKEND KV_OFFLOAD_BACKEND_METADATA TOTAL_CPU_DRAM_GB KV_P2P_TRANSFER \ WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ AIPERF_AGENTIC_CACHE_WARMUP_DURATION AIPERF_UNSAFE_OVERRIDE \ diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh index b3eb0b8fd7..f4e9f84c23 100644 --- a/benchmarks/multi_node/amd_utils/trace_replay.sh +++ b/benchmarks/multi_node/amd_utils/trace_replay.sh @@ -107,6 +107,12 @@ RESULT_FILENAME_BASE="${RESULT_FILENAME:-agentic_bench}" mkdir -p "$RESULT_DIR" +if [ "$PREFILL_ENABLE_DP" = "true" ]; then + set -x + export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true + set +x +fi + resolve_trace_source install_agentic_deps diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0f547cba86..e5a3a81787 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2140,7 +2140,7 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 + image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -2170,7 +2170,7 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: # upstream CI can fetch it; aiperf is built on the fly from # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) # selects the sibling-container mode. - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" decode: num-worker: 1 tp: 8 @@ -2179,32 +2179,32 @@ dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: additional-settings: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=0" - # - spec-decoding: "none" - # conc-list: [ 64, 128 ] - # kv-offloading: dram - # kv-offload-backend: { name: hicache } - # prefill: - # num-worker: 1 - # tp: 8 - # ep: 1 - # dp-attn: true - # additional-settings: - # - "PREFILL_NODES=1" - # # Node-0 sibling client: run the aiperf trace-replay in its own sibling - # # container on node 0 (via the host docker socket) instead of inside the - # # server container. Reuse the (publicly pullable) server image so - # # upstream CI can fetch it; aiperf is built on the fly from - # # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) - # # selects the sibling-container mode. - # - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710" - # decode: - # num-worker: 1 - # tp: 8 - # ep: 1 - # dp-attn: true - # additional-settings: - # - "DECODE_NODES=1" - # - "DECODE_MTP_SIZE=0" + - spec-decoding: "none" + conc-list: [ 64, 96, 128 ] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "PREFILL_NODES=1" + # Node-0 sibling client: run the aiperf trace-replay in its own sibling + # container on node 0 (via the host docker socket) instead of inside the + # server container. Reuse the (publicly pullable) server image so + # upstream CI can fetch it; aiperf is built on the fly from + # /workspace/utils/aiperf. Setting only CLIENT_IMAGE (no CLIENT_NODES) + # selects the sibling-container mode. + - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" # DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM. # EXTRAPOLATED bring-up. Same rationale as dsv4-fp8-mi300x-vllm: sglang has no @@ -2255,3 +2255,4 @@ dsv4-fp8-mi325x-vllm-mtp: # is 55.8%. conc128 passed cleanly on the memory-tightest SKU (MI300X); # cap 8k1k MTP at 128 (normal holds 256, 1k1k holds 512). - { tp: 8, conc-start: 4, conc-end: 128, spec-decoding: mtp } + diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 335245a6c3..04472adb82 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5060,3 +5060,14 @@ - "Re-pin VLLM_ROUTER_IMAGE to vllm/vllm-router:nightly-20260716-1fbcde7 (previous nightly-20260629-e667ebb was garbage-collected from Docker Hub)" - "Exclude known-bad nodes mia1-p01-g09,g14 from the disagg node pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2301 + +- config-keys: + - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache + description: + - "Bump image lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 -> v0.5.15.post1-rocm720-mi35x-20260719." + - "HiCache sizing: recipe now sets HICACHE_RATIO=4 and FORCE_HICACHE_RATIO=1 to explicitly opt into ratio-based sizing (DeepSeek-V4's hybrid HiCache pool rejects --hicache-size); job.slurm now forwards FORCE_HICACHE_RATIO into the container HiCache env file, where it was previously silently dropped." + - "Decode cuda-graph crash workaround: disable prefill cuda-graph for DeepSeek-V4-Pro-AgentX, drop --enable-two-batch-overlap from dp_flags, lower prefill mem_fraction_static 0.8->0.72 and no-dp max_running_requests 128->64; add TORCH_NCCL_BLOCKING_WAIT/NCCL_BLOCKING_WAIT=1 to avoid a ROCm 7.2 HIP-runtime watchdog race during decode cuda-graph capture." + - "Fix router policy env-var bug: server_sglang.sh read ROUTER_PREFILL_POLICY but recipes export PREFILL_ROUTER_POLICY, so the recipe override was silently ignored; also switch the agentic default prefill router policy cache_aware -> consistent_hashing and add --dp-aware (superseding the separate --decode-policy override)." + - "Re-enable the TP8/EP8/DPA (ep=8, dp-attn=true) hicache search-space arm at conc-list [64, 96, 128]." + - "Add RDMA QoS/DCQCN and GPU-sanity pre-flight gates to job.slurm (helpers/rdma_check.sh, helpers/gpu_sanity.sh) to fail fast before burning GPU time on misconfigured NICs or GPU memory held by stale processes." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2308