Merge branch 'main' into amd/agentx_dsv4_sgl_mtp_0717

Author: seungrokj

.github/workflows/collectivex-sweep.yml

@@ -23,6 +23,10 @@ on:
         description: Keep only shards whose expert-parallel degree is in this comma-list (8 keeps EP8 only, dropping EP16 so GB SKUs co-schedule with 8-GPU SKUs; blank = all)
         type: string
         default: ''
+      modes:
+        description: "Keep only shards in this comma-list of modes (normal, low-latency); low-latency is a decode-only EP8 addition; blank = all"
+        type: string
+        default: ''
 concurrency:
   group: cx-${{ github.ref }}-${{ inputs.backend }}-${{ inputs.only_sku }}
   cancel-in-progress: false
@@ -44,12 +48,14 @@ jobs:
           INPUT_ONLY_SKU: ${{ inputs.only_sku }}
           INPUT_EXCLUDE_SKUS: ${{ inputs.exclude_skus }}
           INPUT_EP_SIZES: ${{ inputs.ep_sizes }}
+          INPUT_MODES: ${{ inputs.modes }}
         run: |
           set -euo pipefail
           args=(--backend "$INPUT_BACKEND")
           [ -n "$INPUT_ONLY_SKU" ] && args+=(--only-sku "$INPUT_ONLY_SKU")
           [ -n "$INPUT_EXCLUDE_SKUS" ] && args+=(--exclude-skus "$INPUT_EXCLUDE_SKUS")
           [ -n "$INPUT_EP_SIZES" ] && args+=(--ep-sizes "$INPUT_EP_SIZES")
+          [ -n "$INPUT_MODES" ] && args+=(--modes "$INPUT_MODES")
           python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null
           python3 - "$GITHUB_OUTPUT" <<'PY'
           import json
@@ -94,10 +100,10 @@ jobs:
       COLLX_SHARD_SKU: ${{ matrix.sku }}
       COLLECTIVEX_CANONICAL_GHA: '1'
       COLLECTIVEX_SOURCE_SHA: ${{ github.sha }}
-      # Consolidated shards run one bounded build-group in one Slurm allocation.
-      # MI300X's compute partition has a 180-minute ceiling; the other production
-      # pools accept 300 minutes. Allocations release as soon as the shard finishes.
-      COLLX_TIME: ${{ matrix.sku == 'mi300x' && '180' || '300' }}
+      # Consolidated shards run one bounded build-group in one Slurm allocation;
+      # every production pool accepts 300 minutes. Allocations release as soon as
+      # the shard finishes.
+      COLLX_TIME: '300'
       COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_${{ matrix.id }}
       COLLX_JOB_PARENT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) || '/tmp' }}
       COLLX_JOB_ROOT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) }}

.github/workflows/run-sweep.yml

@@ -581,6 +581,7 @@ jobs:
             kv-offloading: ${{ matrix.config.kv-offloading }}
             kv-offload-backend: ${{ matrix.config['kv-offload-backend'].name }}
             kv-offload-backend-metadata: ${{ matrix.config['kv-offload-backend'] && toJson(matrix.config['kv-offload-backend']) || '' }}
+            total-cpu-dram-gb: ${{ matrix.config['total-cpu-dram-gb'] }}
             duration: ${{ matrix.config.duration }}
             run-eval: false
             scenario-type: agentic-coding

benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh

@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+
+# Agentic trace-replay recipe for a disaggregated SGLang server on MI355X
+# (DeepSeek-V4-Pro FP4, 1P1D TP8).
+#
+# CI-style sibling of dsr1_fp4_mi355x_sglang-disagg.sh: driven entirely by
+# environment variables and submits a SLURM job via submit.sh. The agentic /
+# HiCache-offload configuration mirrors the DSR1 recipe but uses DSV4-Pro
+# specific flags (dsv4 attention backend, page-size 256, SWA settings).
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../../benchmark_lib.sh"
+
+check_env_vars \
+    CONC_LIST \
+    ISL \
+    OSL \
+    IMAGE \
+    SPEC_DECODING \
+    MODEL_PATH \
+    PREFILL_NUM_WORKERS \
+    PREFILL_TP \
+    PREFILL_EP \
+    PREFILL_DP_ATTN \
+    DECODE_NUM_WORKERS \
+    DECODE_TP \
+    DECODE_EP \
+    DECODE_DP_ATTN \
+    PREFILL_NODES \
+    DECODE_NODES \
+    RANDOM_RANGE_RATIO \
+    DURATION \
+    KV_OFFLOADING \
+    IS_AGENTIC \
+    FRAMEWORK
+
+if [[ -n "$SLURM_JOB_ID" ]]; then
+  echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
+fi
+
+set -x
+
+# Use upstreamed multi_node scripts (no external clone needed)
+cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1
+
+# Set up SGL launch script-specific environment variables
+export TIME_LIMIT="${TIME_LIMIT:-08:00:00}"
+export MODEL_PATH=$MODEL_PATH
+export MODEL_NAME=$MODEL_NAME
+export CONTAINER_IMAGE=$IMAGE
+
+# ── Identity / result naming ──
+export MODEL_PREFIX="${MODEL_PREFIX:-dsv4}"
+export PRECISION="${PRECISION:-fp4}"
+export RESULT_FILENAME="${RESULT_FILENAME:-${RUNNER_NAME:-dsv4-fp4-agentic}}"
+
+# ── Agentic benchmark params ──
+export DURATION="${DURATION:-1800}"
+# DSV4-Pro max model len for agentic traces (matches single-node recipe).
+export MAX_MODEL_LEN="${MAX_MODEL_LEN:-1000000}"
+
+# ── In-tree sglang patches ──
+# mori_conn.py targets hybrid-state bugs (GLM-5, Qwen3.5). DSV4-Pro uses a
+# pure MoE/DSA architecture without hybrid state; skip to avoid interference.
+export MORI_CONN_PATCH="${MORI_CONN_PATCH:-skip}"
+
+# ── Aiter fault mitigation ──
+# --disable-custom-all-reduce avoids a known aiter fault on MI355X.
+export DISABLE_CUSTOM_ALL_REDUCE="${DISABLE_CUSTOM_ALL_REDUCE:-0}"
+
+# ── KV cache offloading (HiCache) ──
+# KV_OFFLOADING=none | dram (passed from YAML; default none for disagg).
+# KV_OFFLOAD_BACKEND selects the backend when offloading is on; this recipe
+# only implements HiCache, so "hicache" is the only supported value.
+# HICACHE_TIER: L2 -> GPU + CPU-DRAM host pool. L3 -> + Mooncake store.
+export KV_OFFLOADING="${KV_OFFLOADING:-none}"
+if [[ "$KV_OFFLOADING" != "none" ]]; then
+  export KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-hicache}"
+fi
+# HiCache/Mooncake tunables only matter when KV offloading is enabled.
+if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then
+  export HICACHE_TIER="${HICACHE_TIER:-L2}"
+  export HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-1}"
+  # DSV4 uses page-size 256 (set in models.yaml); HiCache must match.
+  export HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-256}"
+  # HiCache ratio (host pool = ratio * GPU KV pool). Default derived in server_sglang.sh.
+  export HICACHE_RATIO="${HICACHE_RATIO:-}"
+
+  # ── HiCache layout/backend by tier ──
+  #   L3 (Mooncake): page_first + direct + write_through     + storage=mooncake
+  #   L2 (CPU DRAM): layer_first + direct + write_through_selective + storage=none
+  # NOTE: write_through_selective evicts only under GPU memory pressure, avoiding
+  # the mori RDMA race that causes GPU memory access faults with write_through.
+  if [[ "${HICACHE_TIER^^}" == "L3" ]]; then
+    export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}"
+    export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}"
+    export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}"
+    export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-mooncake}"
+  else
+    export HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first}"
+    export HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}"
+    export HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}"
+    export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}"
+  fi
+  export HICACHE_PREFETCH_POLICY="${HICACHE_PREFETCH_POLICY:-best_effort}"
+  # Shared nodes: use non-default Mooncake ports to avoid collisions.
+  export MC_MASTER_PORT="${MC_MASTER_PORT:-58137}"
+  export MC_METADATA_PORT="${MC_METADATA_PORT:-8080}"
+  export MC_METRICS_PORT="${MC_METRICS_PORT:-19003}"
+  export MC_MASTER_THREADS="${MC_MASTER_THREADS:-64}"
+  export MC_EVICTION_HIGH_WATERMARK="${MC_EVICTION_HIGH_WATERMARK:-0.95}"
+  export MC_PATCH_HOSTPOOL="${MC_PATCH_HOSTPOOL:-1}"
+  export MC_PROTOCOL="${MC_PROTOCOL:-tcp}"
+  export MC_GLOBAL_SEG="${MC_GLOBAL_SEG:-64gb}"
+  export MC_DEVICE="${MC_DEVICE:-}"
+  export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}"
+  export MC_METADATA_SERVER="${MC_METADATA_SERVER:-}"
+fi
+
+# ── MoRIIO RDMA Send Queue tuning ──
+export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}"
+export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-32768}"
+
+# ── SGLang PD router policy + server metrics ──
+export PREFILL_ROUTER_POLICY="${PREFILL_ROUTER_POLICY:-cache_aware}"
+export ENABLE_METRICS="${ENABLE_METRICS:-1}"
+
+# ── MTP ──
+export DECODE_MTP_SIZE="${DECODE_MTP_SIZE:-0}"
+
+# Derive EP/DP enable flags from the topology inputs.
+if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then
+export PREFILL_ENABLE_EP=false
+else
+export PREFILL_ENABLE_EP=true
+fi
+
+if [[ "$PREFILL_DP_ATTN" == "true" ]]; then
+export PREFILL_ENABLE_DP=true
+else
+export PREFILL_ENABLE_DP=false
+fi
+
+if [[ "${DECODE_EP:-1}" -eq 1 ]]; then
+export DECODE_ENABLE_EP=false
+else
+export DECODE_ENABLE_EP=true
+fi
+
+if [[ "$DECODE_DP_ATTN" == "true" ]]; then
+export DECODE_ENABLE_DP=true
+else
+export DECODE_ENABLE_DP=false
+fi
+
+# Launch the job. CONC_LIST is space-delimited in YAML; submit.sh wants 'x'.
+JOB_ID=$(bash ./submit.sh $PREFILL_NODES \
+    $PREFILL_NUM_WORKERS \
+    $DECODE_NODES \
+    $DECODE_NUM_WORKERS \
+    $ISL $OSL "${CONC_LIST// /x}" inf \
+    ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \
+    ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \
+    ${PREFILL_TP} ${DECODE_TP} \
+    ${RANDOM_RANGE_RATIO})
+
+if [[ $? -ne 0 ]]; then
+    echo "Failed to submit job" >&2
+    exit 1
+fi
+
+echo "$JOB_ID"

benchmarks/multi_node/amd_utils/env.sh

@@ -12,6 +12,23 @@ set -x
 ENGINE="${ENGINE:-sglang-disagg}"
 export PYTHONDONTWRITEBYTECODE=1
 
+# =============================================================================
+# HiCache / Mooncake settings from job.slurm
+# =============================================================================
+# job.slurm writes the recipe-provided HiCache/Mooncake tunables to
+# hicache_mc_<JID>.env and mounts it read-only at /config/hicache_mc.env. Source
+# it here (auto-export) so values like HICACHE_PAGE_SIZE=256 reach the container
+# before server_sglang.sh applies its "${VAR:-default}" fallbacks. Without this
+# the vars arrive unset and server_sglang.sh defaults HICACHE_PAGE_SIZE to 1,
+# overriding the recipe's --page-size. Empty values in the file are harmless:
+# the "${VAR:-default}" fallbacks still treat "" as unset.
+if [[ -f /config/hicache_mc.env ]]; then
+    set -a
+    source /config/hicache_mc.env
+    set +a
+    echo "[env.sh] sourced HiCache config from /config/hicache_mc.env (HICACHE_PAGE_SIZE=${HICACHE_PAGE_SIZE:-unset})"
+fi
+
 # =============================================================================
 # Shared: IBDEVICES detection
 # =============================================================================
@@ -50,11 +67,11 @@ export NCCL_IB_HCA=${NCCL_IB_HCA:-$IBDEVICES}
 # =============================================================================
 # Shared by the vLLM MoRIIOConnector and the SGLang/MoRI KV-transfer path.
 
-export MORI_IO_SQ_BACKOFF_TIMEOUT_US=50000
-export MORI_IO_QP_MAX_SEND_WR=16384
-export MORI_IO_QP_MAX_CQE=32768
-export MORI_IO_QP_MAX_SGE=2
-export MORI_IO_TC_DISABLE=0
+export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-50000}"
+export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}"
+export MORI_IO_QP_MAX_CQE="${MORI_IO_QP_MAX_CQE:-32768}"
+export MORI_IO_QP_MAX_SGE="${MORI_IO_QP_MAX_SGE:-2}"
+export MORI_IO_TC_DISABLE="${MORI_IO_TC_DISABLE:-0}"
 
 # QoS/DSCP configuration
 # Priority order: 1) Set by runner, 2) Detect via nicctl, 3) Detect from hostname
@@ -197,14 +214,27 @@ else
     export AITER_LOG_LEVEL=ERROR
 
     export SGLANG_MORI_DISPATCH_DTYPE=auto
-    export MORI_COMBINE_DTYPE_PREFILL=fp8_direct_cast
-    export MORI_COMBINE_DTYPE_DECODE=fp8
+    # export MORI_COMBINE_DTYPE_PREFILL=fp8_direct_cast
+    # export MORI_COMBINE_DTYPE_DECODE=fp8
+    export MORI_COMBINE_DTYPE_PREFILL=""
+    export MORI_COMBINE_DTYPE_DECODE=""
     export SGLANG_MORI_QP_PER_TRANSFER=4
     export SGLANG_MORI_NUM_WORKERS=4
+    # Keep these as overridable defaults (not hard assignments), otherwise
+    # later tuning blocks cannot raise them for high-concurrency runs.
+    # export MORI_IO_SQ_BACKOFF_TIMEOUT_US="${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-500000}"
+
+    # export MORI_IO_QP_MAX_SEND_WR="${MORI_IO_QP_MAX_SEND_WR:-16384}"
+    # export MORI_IO_QP_MAX_CQE=32768
+    # export MORI_IO_QP_MAX_SGE=1
+
+    # export MORI_IO_TC_DISABLE=0
 
     export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600
     export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600
 
+    export SGLANG_HEALTH_CHECK_TIMEOUT=600
+
     # GLM-5: uses NSA (not MLA), needs fused-decode-MLA disabled + fast loading
     if [[ "$MODEL_NAME" == "GLM-5-FP8" ]]; then
         export SGLANG_ROCM_FUSED_DECODE_MLA=0
@@ -242,15 +272,34 @@ else
     # 1 mirrors router logs to stdout via tee (useful for live debugging).
     export SGLANG_ROUTER_STDOUT_LOGS="${SGLANG_ROUTER_STDOUT_LOGS:-0}"
 
+    # FIXME: WA for latest upstream 0305 image
+    export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH}
+
     # =========================================================================
     # DeepSeek-V4-Pro PD recipe overrides
     # Placed at the end of the SGLang env block so it wins over the global
     # MoRI/SGLang defaults set above. Mirrors the validated DSv4 manual PD
-    # commands (see dsv4_mi355x_sglang_disagg_plan.md §2). Only the SGLang/MoRI
-    # env knobs are pinned here; CLI flags live in models.yaml and the cluster
-    # NIC/socket vars (NCCL_IB_HCA, *_SOCKET_IFNAME, IBDEVICES) stay runner-derived.
+    # commands (ported from InferenceX amd/dsv4_sgl_di). These SGLANG_OPT_* /
+    # AITER_* kernel-routing knobs steer DSv4 away from the default aiter CK
+    # fused-MoE path, which raises "Unsupported kernel config for moe heuristic
+    # dispatch" at decode time on this fp4 model (job 19034 crash). Only the
+    # SGLang/MoRI env knobs are pinned here; CLI flags live in models.yaml and
+    # the cluster NIC/socket vars stay runner-derived.
     # =========================================================================
     if [[ "$MODEL_NAME" == "DeepSeek-V4-Pro" ]]; then
+        # MoRI RDMA send-queue depth for DSv4 (overrides the global default above).
+        export MORI_IO_QP_MAX_SEND_WR=32767
+        # Unified radix tree: cache impl with per-component (full-attn / SWA)
+        # management for hybrid-attention models. Set unconditionally (not gated on
+        # hicache) so all SGLang runs use it.
+        export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1
+        # Proactively free out-of-window SWA KV slots during chunked prefill.
+        # Without it, in-flight requests pin SWA KV for their whole context, keeping
+        # the SWA pool under constant eviction pressure; under LRU the trailing
+        # window of cached sessions gets flushed, making prefix-cache hits bimodal
+        # and collapsing the effective hit rate on multi-turn agentic workloads.
+        export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1
+
         # MoRI dispatch/combine dtypes: auto for both roles (not the fp8 split default)
         export SGLANG_MORI_DISPATCH_DTYPE=auto
         export MORI_COMBINE_DTYPE_PREFILL=auto
@@ -262,42 +311,44 @@ else
         unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL
         unset MORI_MOE_MAX_INPUT_TOKENS_DECODE
 
-        # PER_RANK dispatch tokens are pinned independently of the sizing above
-        # (16384 prefill / 128 decode in the reference recipe). server_sglang.sh
-        # prefers these over the MORI_MAX_DISPATCH_TOKENS_* coupling when set.
+        # PER_RANK dispatch tokens pinned independently (16384 prefill / 128
+        # decode); server_sglang.sh prefers these over the MORI_MAX_DISPATCH_*
+        # coupling when set.
         export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL=16384
         export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE=128
 
-        # Fixed inter-kernel switch threshold (not derived). NOTE: the DP+EP path in
-        # server_sglang.sh recomputes this dynamically for the DEP topology.
+        # Fixed inter-kernel switch threshold (not derived).
         export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=4096
 
         # Overlap plan stream on for DSv4 (global default is 0)
-        export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
+        # export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0
 
         # DSv4 model kernel routing (mirrors the single-node / manual PD recipe)
         export SGLANG_DEFAULT_THINKING=1
-        export SGLANG_DSV4_REASONING_EFFORT=max
-        export SGLANG_USE_ROCM700A=0
-        export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
+        export SGLANG_DSV4_REASONING_EFFORT=high
         export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false
+        export SGLANG_USE_AITER=1
+        export SGLANG_USE_ROCM700A=0
         export SGLANG_OPT_USE_FUSED_COMPRESS=true
-        export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true
+        export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
         export SGLANG_OPT_FP8_WO_A_GEMM=false
         export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false
         export SGLANG_OPT_USE_TOPK_V2=false
-        export SGLANG_OPT_USE_AITER_INDEXER=true
+        export SGLANG_OPT_USE_AITER_INDEXER=${SGLANG_OPT_USE_AITER_INDEXER:-true}
         export SGLANG_OPT_USE_TILELANG_INDEXER=false
         export SGLANG_OPT_USE_TILELANG_MHC_PRE=false
         export SGLANG_OPT_USE_TILELANG_MHC_POST=false
         export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1
+        export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true
         export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false
         export SGLANG_ROCM_USE_MULTI_STREAM=false
         export AITER_BF16_FP8_MOE_BOUND=0
         export SGLANG_EAGER_INPUT_NO_COPY=true
+        export SGLANG_SHARED_EXPERT_TP1=1
+        export SGLANG_DP_SHARED_EXPERT_LOCAL=1
+        export SGLANG_DP_USE_GATHERV=1
+        export SGLANG_DP_USE_REDUCE_SCATTER=1
+        export GPU_MAX_HW_QUEUES=5
     fi
 
-    # FIXME: WA for latest upstream 0305 image
-    export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH}
-
-fi
+fi