Fix GB300 DeepSeek-V4 AgentX worker metrics / 修复 GB300 DeepSeek-V4 AgentX worker 指标采集

benchmarks/multi_node/agentic_srt.sh

@@ -29,6 +29,30 @@ done
 resolve_trace_source
 install_agentic_deps
 
+# NVIDIA/srt-slurm PR #276 supplies authoritative logical-worker endpoints to
+# custom benchmarks. Keep the local reconstruction as a fallback for tagged
+# upstream releases, and log the selected path so the live validation proves
+# which implementation produced the endpoint list.
+if [ -n "${AIPERF_SERVER_METRICS_URLS:-}" ]; then
+    echo "Using AIPerf worker metrics endpoints supplied by srt-slurm: $AIPERF_SERVER_METRICS_URLS"
+elif [ -n "${AIPERF_SRT_WORKER_NODE_COUNTS:-}" ]; then
+    check_env_vars AIPERF_SRT_INFRA_NODE_COUNT AIPERF_SRT_SYSTEM_PORT_BASE
+    slurm_job_nodelist="${SLURM_JOB_NODELIST:-${SLURM_NODELIST:-}}"
+    if [ -z "$slurm_job_nodelist" ]; then
+        echo "ERROR: SLURM_JOB_NODELIST or SLURM_NODELIST is required to discover worker metrics" >&2
+        exit 1
+    fi
+    export AIPERF_SERVER_METRICS_URLS
+    AIPERF_SERVER_METRICS_URLS=$(
+        "$AIPERF_PYTHON" "$INFMAX_CONTAINER_WORKSPACE/utils/agentic/srt_metrics_endpoints.py" \
+            --nodelist "$slurm_job_nodelist" \
+            --worker-node-counts "$AIPERF_SRT_WORKER_NODE_COUNTS" \
+            --infra-node-count "$AIPERF_SRT_INFRA_NODE_COUNT" \
+            --system-port-base "$AIPERF_SRT_SYSTEM_PORT_BASE"
+    )
+    echo "Discovered AIPerf SGLang worker metrics endpoints: $AIPERF_SERVER_METRICS_URLS"
+fi
+
 wait_for_agentic_servers_idle() {
     local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}"
     local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS:-10}"
@@ -77,6 +101,8 @@ while time.monotonic() < deadline:
             worker_metrics = fetch_metrics(worker_url)
             worker_active += metric_sum(worker_metrics, "vllm:num_requests_running")
             worker_active += metric_sum(worker_metrics, "vllm:num_requests_waiting")
+            worker_active += metric_sum(worker_metrics, "sglang:num_running_reqs")
+            worker_active += metric_sum(worker_metrics, "sglang:num_queue_reqs")
         print(
             f"Agentic drain status: frontend_active={frontend_active:g} "
             f"worker_running_or_waiting={worker_active:g}",

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml

@@ -132,6 +132,11 @@ benchmark:
     INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
     RESULT_DIR: /logs/agentic
     PORT: "8000"
+    # Upstream srt-slurm passes custom benchmark env verbatim. Describe its
+    # topology so agentic_srt.sh can discover the one logical worker leader.
+    AIPERF_SRT_WORKER_NODE_COUNTS: "1"
+    AIPERF_SRT_INFRA_NODE_COUNT: "1"
+    AIPERF_SRT_SYSTEM_PORT_BASE: "7500"
     # Aggregated: one worker serves both prefill and decode, so GPU accounting
     # is the single-worker form num_gpus = TP (not the disagg prefill+decode
     # sum). The multinode post-processing path assumes disagg and would divide
@@ -151,4 +156,3 @@ benchmark:
     # Persistent HF hub cache (also via default_mounts) so the trace dataset
     # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE.
     HF_HUB_CACHE: "/hf_hub_cache"
-

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml

@@ -205,8 +205,12 @@ benchmark:
     INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
     RESULT_DIR: /logs/agentic
     PORT: "8000"
+    # Logical worker node widths in upstream topology order: 6P x 2 nodes,
+    # then 1D x 4 nodes. Ports advance once per physical worker process.
+    AIPERF_SRT_WORKER_NODE_COUNTS: "2,2,2,2,2,2,4"
+    AIPERF_SRT_INFRA_NODE_COUNT: "1"
+    AIPERF_SRT_SYSTEM_PORT_BASE: "7500"
     IS_MULTINODE: "true"
     AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1"
     AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
     HF_HUB_CACHE: "/hf_hub_cache"
-

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml

@@ -204,8 +204,12 @@ benchmark:
     INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
     RESULT_DIR: /logs/agentic
     PORT: "8000"
+    # Logical worker node widths in upstream topology order: 1P x 2 nodes,
+    # then 1D x 1 node. Ports advance once per physical worker process.
+    AIPERF_SRT_WORKER_NODE_COUNTS: "2,1"
+    AIPERF_SRT_INFRA_NODE_COUNT: "1"
+    AIPERF_SRT_SYSTEM_PORT_BASE: "7500"
     IS_MULTINODE: "true"
     AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1"
     AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
     HF_HUB_CACHE: "/hf_hub_cache"
-

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml

@@ -205,8 +205,12 @@ benchmark:
     INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
     RESULT_DIR: /logs/agentic
     PORT: "8000"
+    # Logical worker node widths in upstream topology order: 1P x 2 nodes,
+    # then 1D x 4 nodes. Ports advance once per physical worker process.
+    AIPERF_SRT_WORKER_NODE_COUNTS: "2,4"
+    AIPERF_SRT_INFRA_NODE_COUNT: "1"
+    AIPERF_SRT_SYSTEM_PORT_BASE: "7500"
     IS_MULTINODE: "true"
     AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1"
     AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
     HF_HUB_CACHE: "/hf_hub_cache"
-

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml

@@ -205,8 +205,12 @@ benchmark:
     INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
     RESULT_DIR: /logs/agentic
     PORT: "8000"
+    # Logical worker node widths in upstream topology order: 2P x 2 nodes,
+    # then 1D x 4 nodes. Ports advance once per physical worker process.
+    AIPERF_SRT_WORKER_NODE_COUNTS: "2,2,4"
+    AIPERF_SRT_INFRA_NODE_COUNT: "1"
+    AIPERF_SRT_SYSTEM_PORT_BASE: "7500"
     IS_MULTINODE: "true"
     AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1"
     AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
     HF_HUB_CACHE: "/hf_hub_cache"
-

benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml

@@ -205,8 +205,12 @@ benchmark:
     INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
     RESULT_DIR: /logs/agentic
     PORT: "8000"
+    # Logical worker node widths in upstream topology order: 3P x 2 nodes,
+    # then 1D x 4 nodes. Ports advance once per physical worker process.
+    AIPERF_SRT_WORKER_NODE_COUNTS: "2,2,2,4"
+    AIPERF_SRT_INFRA_NODE_COUNT: "1"
+    AIPERF_SRT_SYSTEM_PORT_BASE: "7500"
     IS_MULTINODE: "true"
     AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1"
     AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
     HF_HUB_CACHE: "/hf_hub_cache"
-

configs/runners.yaml

@@ -315,6 +315,9 @@ hardware:
   cluster:gb200-nv:
     available-cpu-dram-mib: 860_160
     gpus-per-node: 4
+  cluster:gb300-nv:
+    available-cpu-dram-mib: 860_160
+    gpus-per-node: 4
   cluster:mi300x-amds:
     available-cpu-dram-mib: 2_321_924
     gpus-per-node: 8

perf-changelog.yaml

@@ -4974,3 +4974,11 @@
     - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity"
     - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280
+
+- config-keys:
+    - dsv4-fp4-gb300-dynamo-sglang-agentic-agg
+    - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg
+  description:
+    - "Correctly capture AIPerf Prometheus metrics from the Dynamo frontend and every logical SGLang worker endpoint while using unmodified upstream srt-slurm"
+    - "Register GB300 hardware metadata required to generate AgentX DRAM-offload configs"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2291

runners/launch_gb300-nv.sh

@@ -139,13 +139,19 @@ SRTCTL_SETUP_SCRIPT=""
 rm -rf "$SRT_REPO_DIR"
 
 if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then
-    # DSv4 GB300 sglang agentic: NVIDIA/srt-slurm v1.0.10 has the nginx
-    # client_max_body_size fix (>1 MiB agentic warmup bodies), the
-    # session-affinity frontend, and the BenchmarkType.CUSTOM / extra_mount
-    # schema these recipes need.
+    # Directly validate NVIDIA/srt-slurm PR #276, which exposes authoritative
+    # logical worker metrics endpoints to custom benchmarks. Fetch the PR
+    # through NVIDIA's pull ref and verify its immutable head before checkout;
+    # no downstream patch or custom-fork clone is applied.
+    SRT_ENDPOINTS_PR_HEAD="3fe550861fee69d8d3823c9a06aa7c2da2916b82"
     git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
     cd "$SRT_REPO_DIR"
-    git checkout v1.0.10
+    git fetch origin pull/276/head
+    if [[ "$(git rev-parse FETCH_HEAD)" != "$SRT_ENDPOINTS_PR_HEAD" ]]; then
+        echo "ERROR: NVIDIA/srt-slurm PR #276 head changed; refusing to test an unreviewed commit" >&2
+        exit 1
+    fi
+    git checkout --detach "$SRT_ENDPOINTS_PR_HEAD"
     mkdir -p recipes/sglang/deepseek-v4/agentic
     cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \
         recipes/sglang/deepseek-v4/agentic

utils/agentic/srt_metrics_endpoints.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+"""Discover logical srt-slurm worker metrics endpoints for custom benchmarks."""
+
+from __future__ import annotations
+
+import argparse
+import re
+
+
+_NUMERIC_RANGE = re.compile(r"^(\d+)-(\d+)(?::(\d+))?$")
+
+
+def _split_top_level(value: str) -> list[str]:
+    """Split a Slurm hostlist on commas outside bracket expressions."""
+    parts: list[str] = []
+    start = 0
+    depth = 0
+    for index, character in enumerate(value):
+        if character == "[":
+            depth += 1
+        elif character == "]":
+            depth -= 1
+            if depth < 0:
+                raise ValueError(f"unmatched closing bracket in Slurm nodelist: {value!r}")
+        elif character == "," and depth == 0:
+            parts.append(value[start:index])
+            start = index + 1
+    if depth != 0:
+        raise ValueError(f"unmatched opening bracket in Slurm nodelist: {value!r}")
+    parts.append(value[start:])
+    if any(not part for part in parts):
+        raise ValueError(f"empty component in Slurm nodelist: {value!r}")
+    return parts
+
+
+def _expand_bracket_values(value: str) -> list[str]:
+    """Expand the comma-separated numeric values inside one bracket pair."""
+    expanded: list[str] = []
+    for component in value.split(","):
+        match = _NUMERIC_RANGE.fullmatch(component)
+        if match is None:
+            if not component.isdigit():
+                raise ValueError(f"unsupported Slurm bracket component: {component!r}")
+            expanded.append(component)
+            continue
+
+        start_text, end_text, step_text = match.groups()
+        start = int(start_text)
+        end = int(end_text)
+        step = int(step_text or "1")
+        if step == 0:
+            raise ValueError(f"Slurm range step must be positive: {component!r}")
+        direction = 1 if end >= start else -1
+        width = max(len(start_text), len(end_text))
+        stop = end + direction
+        expanded.extend(f"{number:0{width}d}" for number in range(start, stop, direction * step))
+    return expanded
+
+
+def _expand_component(component: str) -> list[str]:
+    """Recursively expand all bracket expressions in one hostlist component."""
+    opening = component.find("[")
+    if opening < 0:
+        if "]" in component:
+            raise ValueError(f"unmatched closing bracket in Slurm nodelist component: {component!r}")
+        return [component]
+
+    depth = 0
+    closing = -1
+    for index in range(opening, len(component)):
+        if component[index] == "[":
+            depth += 1
+        elif component[index] == "]":
+            depth -= 1
+            if depth == 0:
+                closing = index
+                break
+    if closing < 0:
+        raise ValueError(f"unmatched opening bracket in Slurm nodelist component: {component!r}")
+
+    prefix = component[:opening]
+    values = _expand_bracket_values(component[opening + 1 : closing])
+    suffixes = _expand_component(component[closing + 1 :])
+    return [f"{prefix}{value}{suffix}" for value in values for suffix in suffixes]
+
+
+def expand_slurm_nodelist(nodelist: str) -> list[str]:
+    """Expand a Slurm nodelist without relying on ``scontrol`` in the container."""
+    if not nodelist:
+        raise ValueError("Slurm nodelist must not be empty")
+    hosts: list[str] = []
+    for component in _split_top_level(nodelist):
+        hosts.extend(_expand_component(component))
+    return hosts
+
+
+def parse_worker_node_counts(value: str) -> list[int]:
+    """Parse comma-separated physical node counts for logical workers."""
+    try:
+        counts = [int(component) for component in value.split(",")]
+    except ValueError as error:
+        raise ValueError(f"worker node counts must be comma-separated integers: {value!r}") from error
+    if not counts or any(count < 1 for count in counts):
+        raise ValueError(f"every logical worker must use at least one node: {value!r}")
+    return counts
+
+
+def build_worker_metrics_urls(
+    nodelist: str,
+    worker_node_counts: list[int],
+    infra_node_count: int,
+    system_port_base: int,
+) -> list[str]:
+    """Map logical worker leaders to upstream srt-slurm system ports."""
+    if infra_node_count < 0:
+        raise ValueError("infra node count must not be negative")
+    if system_port_base < 1 or system_port_base > 65535:
+        raise ValueError(f"invalid system port base: {system_port_base}")
+
+    allocated_hosts = expand_slurm_nodelist(nodelist)
+    expected_nodes = infra_node_count + sum(worker_node_counts)
+    if len(allocated_hosts) != expected_nodes:
+        raise ValueError(
+            "Slurm allocation does not match the custom benchmark topology: "
+            f"expanded {len(allocated_hosts)} hosts, expected {expected_nodes} "
+            f"({infra_node_count} infra + {sum(worker_node_counts)} worker nodes)"
+        )
+
+    worker_hosts = allocated_hosts[infra_node_count:]
+    urls: list[str] = []
+    process_offset = 0
+    for node_count in worker_node_counts:
+        port = system_port_base + process_offset
+        if port > 65535:
+            raise ValueError(f"derived system port exceeds 65535: {port}")
+        urls.append(f"http://{worker_hosts[process_offset]}:{port}/metrics")
+        process_offset += node_count
+    return urls
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--nodelist", required=True)
+    parser.add_argument("--worker-node-counts", required=True)
+    parser.add_argument("--infra-node-count", required=True, type=int)
+    parser.add_argument("--system-port-base", required=True, type=int)
+    args = parser.parse_args()
+
+    urls = build_worker_metrics_urls(
+        nodelist=args.nodelist,
+        worker_node_counts=parse_worker_node_counts(args.worker_node_counts),
+        infra_node_count=args.infra_node_count,
+        system_port_base=args.system_port_base,
+    )
+    print(",".join(urls))
+
+
+if __name__ == "__main__":
+    main()