[AMD][AgentX] DSv4 MI355X agentic disagg: add high-concurrency dots using DEP8

benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh

@@ -83,8 +83,20 @@ if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; t
   export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}"
   # DSV4 uses page-size 256 (set in models.yaml); HiCache must match.
   export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}"
-  # HiCache ratio (host pool = ratio * GPU KV pool). Default derived in server_sglang.sh.
-  export HICACHE_RATIO="${HICACHE_RATIO:-}"
+  # HiCache ratio (host pool = ratio * GPU KV pool).
+  export HICACHE_RATIO="${HICACHE_RATIO:-4}"
+  # server_sglang.sh prefers an absolute --hicache-size (derived from
+  # TOTAL_CPU_DRAM_GB, the sweep generator's per-node DRAM budget) over
+  # --hicache-ratio whenever TOTAL_CPU_DRAM_GB is set. DSv4 wants the
+  # ratio-based pool instead. Use FORCE_HICACHE_RATIO to opt out of the
+  # --hicache-size path rather than unsetting TOTAL_CPU_DRAM_GB itself:
+  # that var is also the shared client-side gate (benchmark_lib.sh requires
+  # it to be a positive integer whenever KV_OFFLOADING=dram) and gets
+  # forwarded into the aiperf sibling container's client.env, so unsetting
+  # it here made the client fail its own env validation before benchmarking
+  # ("DRAM KV offloading requires a positive configured TOTAL_CPU_DRAM_GB
+  # capacity") even though the servers came up fine.
+  export FORCE_HICACHE_RATIO=1
 
   # ── HiCache layout/backend by tier ──
   #   L3 (Mooncake): page_first + direct + write_through     + storage=mooncake
@@ -122,7 +134,7 @@ export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}"
 export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-32768}"
 
 # ── SGLang PD router policy + server metrics ──
-export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-cache_aware}"
+export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-consistent_hashing}"
 export ENABLE_METRICS="${ENABLE_METRICS:-1}"
 
 # ── MTP ──

benchmarks/multi_node/amd_utils/env.sh

@@ -275,6 +275,26 @@ else
     # FIXME: WA for latest upstream 0305 image
     export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH}
 
+    # Decode CUDA-graph capture crash on ROCm 7.2.0 (TP8+EP8, mori a2a).
+    # Symptom: during decode cuda-graph capture, the torch ProcessGroupNCCL
+    # *watchdog* thread calls hipEventQuery() to poll in-flight NCCL work.
+    # ROCm <= 7.2.0's HIP runtime does NOT honor cudaStreamCaptureModeThreadLocal,
+    # so the watchdog's cross-thread query touches the main thread's active
+    # capture and invalidates it -> "HIP error: operation not permitted on an
+    # event last recorded in a capturing stream (hipErrorCapturedEvent)" ->
+    # watchdog aborts -> "Rank 0 scheduler died during initialization" (-6).
+    # This is a HIP runtime bug, not OOM and not a mori/deepep-mode bug (it
+    # fires for --deepep-mode normal and auto alike; EP8+mori just adds NCCL
+    # PGs that make the watchdog race fire). Refs: sgl-project/sglang#29235,
+    # #24011; ROCm/hip#3876; pytorch/pytorch#176251.
+    # Real fix = ROCm 7.2.2+ (honors THREAD_LOCAL). Until the base image is
+    # bumped, TORCH_NCCL_BLOCKING_WAIT=true makes NCCL work completion use a
+    # blocking wait instead of the async watchdog hipEventQuery poll, so no
+    # event is queried during capture. CUDA graph stays fully enabled.
+    export TORCH_NCCL_BLOCKING_WAIT="${TORCH_NCCL_BLOCKING_WAIT:-1}"
+    export NCCL_BLOCKING_WAIT="${NCCL_BLOCKING_WAIT:-1}"
+    # export NCCL_DEBUG="${NCCL_DEBUG:-INFO}"
+
     # =========================================================================
     # DeepSeek-V4-Pro PD recipe overrides
     # Placed at the end of the SGLang env block so it wins over the global

benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# GPU sanity gate, run on each node AFTER all containers are stopped.
+#
+# At that point VRAM should fall back to driver baseline. Any large remaining
+# usage means a bare (non-containerized) process is hogging the GPU -- which
+# would otherwise surface as an opaque "memory capacity is unbalanced" OOM in
+# the middle of model load ~30 min into the job. We fail fast and name the
+# offender instead. We only report; we never kill another user's process.
+#
+# Ported from https://github.com/ROCm/InferenceY/blob/main/benchmarks/multi_node/amd_utils/gpu_sanity.sh
+set -uo pipefail
+
+# Fail the node if any single GPU still has >= this many GB used.
+GPU_BUSY_GB="${GPU_BUSY_GB:-8}"
+# Poll a few times to give docker stop time to release VRAM before judging.
+GPU_DRAIN_TRIES="${GPU_DRAIN_TRIES:-6}"
+
+if ! command -v rocm-smi >/dev/null 2>&1; then
+    echo "[gpu-sanity] rocm-smi not found on $(hostname); skipping GPU check" >&2
+    exit 0
+fi
+
+busy_gb=0
+for _ in $(seq "$GPU_DRAIN_TRIES"); do
+    used=$(rocm-smi --showmeminfo vram --json 2>/dev/null \
+        | grep -oE '"VRAM Total Used Memory \(B\)": "[0-9]+"' \
+        | grep -oE '[0-9]+' | sort -n | tail -1)
+    busy_gb=$(( ${used:-0} / 1024 / 1024 / 1024 ))
+    if [ "$busy_gb" -lt "$GPU_BUSY_GB" ]; then
+        exit 0
+    fi
+    sleep 5
+done
+
+echo "FATAL: $(hostname) GPU still has ${busy_gb}GB used after stopping all containers" >&2
+echo "       (threshold ${GPU_BUSY_GB}GB) -- a bare (non-containerized) process is hogging the GPU:" >&2
+rocm-smi --showpids >&2 || true
+exit 1

benchmarks/multi_node/amd_utils/helpers/rdma_check.sh

@@ -0,0 +1,160 @@
+#!/bin/bash
+# Pre-flight RDMA QoS/DCQCN validation for a single node, run by job.slurm via
+# `srun` across every allocated node BEFORE any container/GPU time is spent.
+#
+# A misconfigured NIC (PFC not covering the RoCE priority, DCQCN disabled, ...)
+# doesn't make MoRI's cross-node EP/RDMA transfers error out -- it just quietly
+# degrades, and the only symptom is unexplained tail latency or throughput
+# variance in the benchmark numbers hours later. Failing fast here, before the
+# job burns node-hours, is much cheaper than debugging that after the fact.
+#
+# check_qos()/check_dcqcn() below are trimmed/adapted from ROCm/mori's
+# tools/env_check.sh:
+#   https://github.com/ROCm/mori/blob/main/tools/env_check.sh
+# The upstream script's expensive ib_write_bw/ib_write_lat full-mesh bandwidth
+# and latency tests are intentionally NOT ported here -- this runs before
+# every single job, so it must be fast (seconds), not a multi-minute
+# fabric-wide benchmark of its own.
+#
+# Scoped to AMD Pollara (ionic) NICs via nicctl, matching this repo's current
+# fleet (see env.sh's nicctl-based MORI_RDMA_TC/SL detection). If bnxt_re or
+# mlx5 NICs are added to the fleet, port the bnxt_*/mlx5_* check_* functions
+# from the upstream script following the same pattern.
+#
+# Exit code: 0 = OK (or gracefully skipped, e.g. no ionic NICs on this host),
+#            1 = hard QoS/DCQCN misconfiguration -- do not proceed with the run.
+set -uo pipefail
+
+AINIC_MIN_VER="1.117.5-a-45"   # minimum recommended AINIC firmware for IBGDA
+
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[0;33m'
+NC='\033[0m'
+
+log_ok()   { echo -e "[$(hostname -s)] ${GREEN}[OK]${NC}   $*"; }
+log_fail() { echo -e "[$(hostname -s)] ${RED}[FAIL]${NC} $*"; }
+log_warn() { echo -e "[$(hostname -s)] ${YELLOW}[WARN]${NC} $*"; }
+die()      { log_fail "$@"; exit 1; }
+
+# version_ge <candidate> <min> -> true if <candidate> >= <min> (dotted/hyphenated, via sort -V)
+version_ge() {
+    local cand="$1" min="$2"
+    [[ "$cand" == "$min" ]] && return 0
+    [[ "$(printf '%s\n%s\n' "$cand" "$min" | sort -V | head -1)" == "$min" ]]
+}
+
+# check_ainic_version_recommendation <fw_version>
+#   warns if firmware is on the IBGDA-incapable 1.117.1 branch, or below the
+#   recommended minimum for cross-node MORI (EP over RDMA / IBGDA).
+check_ainic_version_recommendation() {
+    local ver="$1"
+    [[ -n "$ver" ]] || { log_warn "cannot verify AINIC firmware version against recommendation (empty)"; return; }
+    if [[ "$ver" =~ ^1\.117\.1([.-]|$) ]]; then
+        log_warn "AINIC firmware $ver is on the 1.117.1 branch, which does NOT support IBGDA -- upgrade to >= $AINIC_MIN_VER"
+    elif version_ge "$ver" "$AINIC_MIN_VER"; then
+        log_ok "AINIC firmware $ver meets the recommended minimum (>= $AINIC_MIN_VER) for cross-node IBGDA"
+    else
+        log_warn "AINIC firmware $ver is below the recommended minimum (>= $AINIC_MIN_VER) for cross-node IBGDA"
+    fi
+}
+
+# check_versions() -- informational only (never hard-fails the job).
+check_versions() {
+    local fw_output sw_output
+    fw_output=$(sudo nicctl show version firmware 2>/dev/null)
+    sw_output=$(sudo nicctl show version host-software 2>/dev/null)
+
+    local fw_versions fw_count
+    fw_versions=$(echo "$fw_output" | grep -i "firmware" | awk '{print $NF}' | sort -u)
+    fw_count=$(echo "$fw_versions" | grep -c . || true)
+    if [[ "$fw_count" -ne 1 ]]; then
+        log_warn "firmware versions not consistent across NICs:"
+        echo "$fw_versions"
+        local v
+        while read -r v; do [[ -n "$v" ]] && check_ainic_version_recommendation "$v"; done <<< "$fw_versions"
+    else
+        log_ok "firmware : $fw_versions"
+        check_ainic_version_recommendation "$fw_versions"
+    fi
+
+    local nicctl_ver
+    nicctl_ver=$(echo "$sw_output" | grep "nicctl" | awk '{print $NF}')
+    [[ -n "$nicctl_ver" ]] && log_ok "nicctl : $nicctl_ver" || log_warn "cannot determine nicctl version"
+}
+
+# check_qos() -- HARD gate: classification type must be DSCP, and PFC no-drop
+# must be enabled and cover every no-drop priority. Dies (exit 1) otherwise.
+check_qos() {
+    local qos_output
+    qos_output=$(sudo nicctl show qos 2>/dev/null)
+    [[ -n "$qos_output" ]] || die "sudo nicctl show qos returned nothing"
+
+    local class_type
+    class_type=$(echo "$qos_output" | grep "Classification type" | head -1 | awk '{print $NF}')
+    [[ "$class_type" == "DSCP" ]] || die "classification type is '$class_type', expected 'DSCP'"
+    log_ok "classification type : DSCP"
+
+    local nd_prio_raw
+    nd_prio_raw=$(echo "$qos_output" | grep "PFC no-drop priorities" | head -1 | awk '{print $NF}')
+    [[ -n "$nd_prio_raw" ]] || die "cannot find PFC no-drop priority"
+    local nd_prios=()
+    IFS=',' read -ra nd_prios <<< "$nd_prio_raw"
+    log_ok "no-drop priorities : ${nd_prios[*]}"
+
+    local pfc_bitmap
+    pfc_bitmap=$(echo "$qos_output" | grep "PFC priority bitmap" | head -1 | awk '{print $NF}')
+    [[ -n "$pfc_bitmap" && "$pfc_bitmap" != "0x0" ]] || die "PFC is not enabled (bitmap=$pfc_bitmap)"
+    local p
+    for p in "${nd_prios[@]}"; do
+        (( pfc_bitmap & (1 << p) )) || die "PFC bitmap $pfc_bitmap does not cover priority $p"
+    done
+    log_ok "PFC enabled for priorities ${nd_prios[*]} (bitmap=$pfc_bitmap)"
+}
+
+# check_dcqcn() -- HARD gate: DCQCN must be enabled on every ROCE device, and
+# the CNP DSCP must be consistent across NICs. Exits 1 otherwise.
+check_dcqcn() {
+    local dcqcn_output
+    dcqcn_output=$(sudo nicctl show dcqcn 2>/dev/null)
+    [[ -n "$dcqcn_output" ]] || die "sudo nicctl show dcqcn returned nothing"
+
+    local total
+    total=$(echo "$dcqcn_output" | grep -c "ROCE device")
+    [[ "$total" -gt 0 ]] || die "no ROCE devices found in dcqcn output"
+
+    local disabled
+    disabled=$(echo "$dcqcn_output" | grep "Status" | grep -v "Enabled" || true)
+    if [[ -n "$disabled" ]]; then
+        log_fail "some ROCE devices have DCQCN disabled:"
+        echo "$disabled"
+        exit 1
+    fi
+    log_ok "DCQCN enabled on all $total ROCE devices"
+
+    local cnp_values cnp_count
+    cnp_values=$(echo "$dcqcn_output" | grep "DSCP value used for CNP" | awk '{print $NF}' | sort -u)
+    cnp_count=$(echo "$cnp_values" | grep -c . || true)
+    [[ "$cnp_count" -eq 1 ]] || die "CNP DSCP not consistent across NICs: $cnp_values"
+    log_ok "CNP DSCP = $cnp_values (consistent across all NICs)"
+}
+
+# ============================= main =============================
+
+if ! command -v nicctl &>/dev/null; then
+    log_warn "nicctl not found on $(hostname -s) -- skipping RDMA QoS/DCQCN pre-flight check (not an ionic NIC host, or nicctl not on PATH)"
+    exit 0
+fi
+
+# nicctl exits 0 even with no NIC present, so check its output rather than its exit code.
+_nicctl_probe=$(sudo nicctl show version firmware 2>&1 || true)
+if echo "$_nicctl_probe" | grep -qiE 'No AMD NICs|Invalid card handle|Failed to get NIC'; then
+    log_warn "nicctl present but no ionic NIC detected/accessible on $(hostname -s) -- skipping RDMA QoS/DCQCN pre-flight check"
+    exit 0
+fi
+
+check_versions
+check_qos
+check_dcqcn
+
+log_ok "RDMA QoS/DCQCN pre-flight check passed on $(hostname -s)"

benchmarks/multi_node/amd_utils/job.slurm

@@ -345,6 +345,29 @@ export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}"
 
 SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,)
 
+# =============================================================================
+# RDMA QoS / DCQCN Pre-flight Check
+# =============================================================================
+# Gate the run on NIC QoS (PFC/DSCP) and DCQCN config on every node before
+# any container/GPU time is spent. Runs on the bare host (nicctl is a host
+# tool). See rdma_check.sh for details.
+RDMA_CHECK_SCRIPT="$(pwd)/helpers/rdma_check.sh"
+if [[ "${SKIP_RDMA_CHECK:-0}" == "1" ]]; then
+    echo "[INFO] SKIP_RDMA_CHECK=1 set; skipping RDMA QoS/DCQCN pre-flight check"
+elif [[ -f "$RDMA_CHECK_SCRIPT" ]]; then
+    echo "Checking RDMA QoS/DCQCN configuration on all $NUM_NODES allocated node(s)..."
+    srun --nodelist="$SELECTED_NODELIST_SRUN" --ntasks=$NUM_NODES bash "$RDMA_CHECK_SCRIPT"
+    RDMA_CHECK_RC=$?
+    if [[ $RDMA_CHECK_RC -ne 0 ]]; then
+        echo "FATAL: RDMA QoS/DCQCN pre-flight check failed on one or more nodes (see [FAIL] lines above)."
+        echo "       Set SKIP_RDMA_CHECK=1 to bypass (not recommended -- MoRI cross-node transfers would run unprotected by PFC/DCQCN)."
+        exit 1
+    fi
+    echo "RDMA QoS/DCQCN pre-flight check passed on all $NUM_NODES node(s)"
+else
+    echo "[WARN] $RDMA_CHECK_SCRIPT not found; skipping RDMA QoS/DCQCN pre-flight check"
+fi
+
 cleanup() {
   echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning up container + stale logs..."
   # Backstop: on scancel/timeout/step-hang the foreground `exec docker run`
@@ -494,6 +517,7 @@ fi
 HICACHE_MC_CONFIG="${BENCHMARK_LOGS_DIR}/hicache_mc_${SLURM_JOB_ID}.env"
 cat > "$HICACHE_MC_CONFIG" <<EOF
 HICACHE_RATIO=${HICACHE_RATIO:-}
+FORCE_HICACHE_RATIO=${FORCE_HICACHE_RATIO:-}
 HICACHE_HOST_POOL_COUNT=${HICACHE_HOST_POOL_COUNT:-}
 HICACHE_PAGE_SIZE=${HICACHE_PAGE_SIZE:-}
 HICACHE_IO_BACKEND=${HICACHE_IO_BACKEND:-}
@@ -635,6 +659,12 @@ fi  # end: if ENGINE == atom-disagg
 \$DOCKER_CMD ps -aq --filter \"$CONT_FILTER\" | xargs -r \$DOCKER_CMD rm -f || true
 \$DOCKER_CMD ps -aq | xargs -r \$DOCKER_CMD stop || true
 
+# GPU sanity gate: containers are stopped, so any remaining VRAM use is a bare
+# (non-containerized) process hogging the GPU -- fail fast (and name it)
+# instead of OOMing in model load ~30 min later. set -e + --kill-on-bad-exit
+# tears down the whole job on non-zero exit.
+bash \"$DI_REPO_DIR/benchmarks/multi_node/amd_utils/helpers/gpu_sanity.sh\"
+
 # Start vLLM external router container on node 0
 if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then
     \$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true

benchmarks/multi_node/amd_utils/models.yaml

@@ -365,19 +365,19 @@ DeepSeek-R1-0528-MXFP4-v2:
 
 DeepSeek-V4-Pro-AgentX:
   base_flags: "--watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --log-level info --log-level-http error"
-  dp_flags: "--enable-dp-attention --enable-prefill-delayer --enable-two-batch-overlap"
+  dp_flags: "--enable-dp-attention --enable-prefill-delayer"
   ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal"
   prefill:
-    mem_fraction_static: 0.8
+    mem_fraction_static: 0.72
     disable_radix_cache: false
-    disable_cuda_graph: false
+    disable_cuda_graph: true
     dp:
       max_running_requests: 1024
       chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE"  # dsv4 compressor kernel uint16 token cap (255*256)
       context_length: 1048576
       # max_total_tokens: 1048576
     no_dp:
-      max_running_requests: 128
+      max_running_requests: 64
       # Small prefill chunks interleave long-context agentic prefills across
       # requests instead of letting one ~100K-token prefill monopolize the
       # engine (the conc>=16 queue-saturation / decode-stall failure mode).