From a42cc4bf515e4f7f96ffeaaf069f02158b3d199e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 20 Jul 2026 18:06:05 -0500 Subject: [PATCH 1/2] fix(agentic): capture GB300 SGLang worker metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Derive logical worker Prometheus endpoints in the InferenceX custom benchmark from the Slurm allocation and recipe worker widths while keeping NVIDIA/srt-slurm v1.0.10 unmodified. Teach the drain gate SGLang queue metrics, add topology tests, and retain the GB300 hardware metadata needed by matrix generation. 中文：在 InferenceX 自定义基准测试中，根据 Slurm 资源分配和配方中的 worker 节点宽度推导各逻辑 worker 的 Prometheus 端点，保持 NVIDIA/srt-slurm v1.0.10 完全不变。为排空检查补充 SGLang 队列指标，增加拓扑测试，并保留矩阵生成所需的 GB300 硬件元数据。 --- benchmarks/multi_node/agentic_srt.sh | 26 +++ .../agentic/agg-gb300-tp4-mtp-kvoffload.yaml | 6 +- ...-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml | 6 +- ...-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml | 6 +- ...00-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml | 6 +- ...00-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml | 6 +- ...00-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml | 6 +- configs/runners.yaml | 3 + perf-changelog.yaml | 8 + runners/launch_gb300-nv.sh | 4 +- utils/agentic/srt_metrics_endpoints.py | 159 ++++++++++++++++++ utils/agentic/test_srt_metrics_endpoints.py | 91 ++++++++++ 12 files changed, 320 insertions(+), 7 deletions(-) create mode 100644 utils/agentic/srt_metrics_endpoints.py create mode 100644 utils/agentic/test_srt_metrics_endpoints.py diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh index 79a36da524..5b667c51cc 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/multi_node/agentic_srt.sh @@ -29,6 +29,30 @@ done resolve_trace_source install_agentic_deps +# Upstream srt-slurm passes benchmark.env through to custom commands, but only +# its built-in AIPerf runners discover worker metrics endpoints. Reconstruct +# the same endpoint mapping here from the full job allocation. The recipe +# provides one node count per logical worker in upstream topology order; srt- +# slurm assigns DYN_SYSTEM_PORT sequentially to every physical worker process, +# so each logical leader's port is base + its first physical-process offset. +if [ -z "${AIPERF_SERVER_METRICS_URLS:-}" ] && [ -n "${AIPERF_SRT_WORKER_NODE_COUNTS:-}" ]; then + check_env_vars AIPERF_SRT_INFRA_NODE_COUNT AIPERF_SRT_SYSTEM_PORT_BASE + slurm_job_nodelist="${SLURM_JOB_NODELIST:-${SLURM_NODELIST:-}}" + if [ -z "$slurm_job_nodelist" ]; then + echo "ERROR: SLURM_JOB_NODELIST or SLURM_NODELIST is required to discover worker metrics" >&2 + exit 1 + fi + export AIPERF_SERVER_METRICS_URLS + AIPERF_SERVER_METRICS_URLS=$( + "$AIPERF_PYTHON" "$INFMAX_CONTAINER_WORKSPACE/utils/agentic/srt_metrics_endpoints.py" \ + --nodelist "$slurm_job_nodelist" \ + --worker-node-counts "$AIPERF_SRT_WORKER_NODE_COUNTS" \ + --infra-node-count "$AIPERF_SRT_INFRA_NODE_COUNT" \ + --system-port-base "$AIPERF_SRT_SYSTEM_PORT_BASE" + ) + echo "Discovered AIPerf SGLang worker metrics endpoints: $AIPERF_SERVER_METRICS_URLS" +fi + wait_for_agentic_servers_idle() { local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}" local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS:-10}" @@ -77,6 +101,8 @@ while time.monotonic() < deadline: worker_metrics = fetch_metrics(worker_url) worker_active += metric_sum(worker_metrics, "vllm:num_requests_running") worker_active += metric_sum(worker_metrics, "vllm:num_requests_waiting") + worker_active += metric_sum(worker_metrics, "sglang:num_running_reqs") + worker_active += metric_sum(worker_metrics, "sglang:num_queue_reqs") print( f"Agentic drain status: frontend_active={frontend_active:g} " f"worker_running_or_waiting={worker_active:g}", diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml index 3ff9e31924..e50515eea8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml @@ -132,6 +132,11 @@ benchmark: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic PORT: "8000" + # Upstream srt-slurm passes custom benchmark env verbatim. Describe its + # topology so agentic_srt.sh can discover the one logical worker leader. + AIPERF_SRT_WORKER_NODE_COUNTS: "1" + AIPERF_SRT_INFRA_NODE_COUNT: "1" + AIPERF_SRT_SYSTEM_PORT_BASE: "7500" # Aggregated: one worker serves both prefill and decode, so GPU accounting # is the single-worker form num_gpus = TP (not the disagg prefill+decode # sum). The multinode post-processing path assumes disagg and would divide @@ -151,4 +156,3 @@ benchmark: # Persistent HF hub cache (also via default_mounts) so the trace dataset # isn't re-downloaded each run; overrides the workflow-level HF_HUB_CACHE. HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml index 715b0c631e..2269dc0b30 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml @@ -205,8 +205,12 @@ benchmark: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic PORT: "8000" + # Logical worker node widths in upstream topology order: 6P x 2 nodes, + # then 1D x 4 nodes. Ports advance once per physical worker process. + AIPERF_SRT_WORKER_NODE_COUNTS: "2,2,2,2,2,2,4" + AIPERF_SRT_INFRA_NODE_COUNT: "1" + AIPERF_SRT_SYSTEM_PORT_BASE: "7500" IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml index dc91c255ce..7b70be6de2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml @@ -204,8 +204,12 @@ benchmark: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic PORT: "8000" + # Logical worker node widths in upstream topology order: 1P x 2 nodes, + # then 1D x 1 node. Ports advance once per physical worker process. + AIPERF_SRT_WORKER_NODE_COUNTS: "2,1" + AIPERF_SRT_INFRA_NODE_COUNT: "1" + AIPERF_SRT_SYSTEM_PORT_BASE: "7500" IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml index 6df695b914..5d806b2ad5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -205,8 +205,12 @@ benchmark: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic PORT: "8000" + # Logical worker node widths in upstream topology order: 1P x 2 nodes, + # then 1D x 4 nodes. Ports advance once per physical worker process. + AIPERF_SRT_WORKER_NODE_COUNTS: "2,4" + AIPERF_SRT_INFRA_NODE_COUNT: "1" + AIPERF_SRT_SYSTEM_PORT_BASE: "7500" IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml index 107a787ba9..a123a04226 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -205,8 +205,12 @@ benchmark: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic PORT: "8000" + # Logical worker node widths in upstream topology order: 2P x 2 nodes, + # then 1D x 4 nodes. Ports advance once per physical worker process. + AIPERF_SRT_WORKER_NODE_COUNTS: "2,2,4" + AIPERF_SRT_INFRA_NODE_COUNT: "1" + AIPERF_SRT_SYSTEM_PORT_BASE: "7500" IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml index 27a4e42d22..81911c6e3b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml @@ -205,8 +205,12 @@ benchmark: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic PORT: "8000" + # Logical worker node widths in upstream topology order: 3P x 2 nodes, + # then 1D x 4 nodes. Ports advance once per physical worker process. + AIPERF_SRT_WORKER_NODE_COUNTS: "2,2,2,4" + AIPERF_SRT_INFRA_NODE_COUNT: "1" + AIPERF_SRT_SYSTEM_PORT_BASE: "7500" IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/configs/runners.yaml b/configs/runners.yaml index 1386652f5a..c4bb40ba85 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -315,6 +315,9 @@ hardware: cluster:gb200-nv: available-cpu-dram-mib: 860_160 gpus-per-node: 4 + cluster:gb300-nv: + available-cpu-dram-mib: 860_160 + gpus-per-node: 4 cluster:mi300x-amds: available-cpu-dram-mib: 2_321_924 gpus-per-node: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 892e3ee2b6..8db205939d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4974,3 +4974,11 @@ - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity" - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2280 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Correctly capture AIPerf Prometheus metrics from the Dynamo frontend and every logical SGLang worker endpoint while using unmodified upstream srt-slurm" + - "Register GB300 hardware metadata required to generate AgentX DRAM-offload configs" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2291 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index d450cdb86d..8978ec38ff 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -142,7 +142,9 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == # DSv4 GB300 sglang agentic: NVIDIA/srt-slurm v1.0.10 has the nginx # client_max_body_size fix (>1 MiB agentic warmup bodies), the # session-affinity frontend, and the BenchmarkType.CUSTOM / extra_mount - # schema these recipes need. + # schema these recipes need. Keep this checkout byte-for-byte upstream; + # agentic_srt.sh derives custom-benchmark metrics endpoints from the Slurm + # allocation and recipe-provided logical worker widths. git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" git checkout v1.0.10 diff --git a/utils/agentic/srt_metrics_endpoints.py b/utils/agentic/srt_metrics_endpoints.py new file mode 100644 index 0000000000..4f70901120 --- /dev/null +++ b/utils/agentic/srt_metrics_endpoints.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Discover logical srt-slurm worker metrics endpoints for custom benchmarks.""" + +from __future__ import annotations + +import argparse +import re + + +_NUMERIC_RANGE = re.compile(r"^(\d+)-(\d+)(?::(\d+))?$") + + +def _split_top_level(value: str) -> list[str]: + """Split a Slurm hostlist on commas outside bracket expressions.""" + parts: list[str] = [] + start = 0 + depth = 0 + for index, character in enumerate(value): + if character == "[": + depth += 1 + elif character == "]": + depth -= 1 + if depth < 0: + raise ValueError(f"unmatched closing bracket in Slurm nodelist: {value!r}") + elif character == "," and depth == 0: + parts.append(value[start:index]) + start = index + 1 + if depth != 0: + raise ValueError(f"unmatched opening bracket in Slurm nodelist: {value!r}") + parts.append(value[start:]) + if any(not part for part in parts): + raise ValueError(f"empty component in Slurm nodelist: {value!r}") + return parts + + +def _expand_bracket_values(value: str) -> list[str]: + """Expand the comma-separated numeric values inside one bracket pair.""" + expanded: list[str] = [] + for component in value.split(","): + match = _NUMERIC_RANGE.fullmatch(component) + if match is None: + if not component.isdigit(): + raise ValueError(f"unsupported Slurm bracket component: {component!r}") + expanded.append(component) + continue + + start_text, end_text, step_text = match.groups() + start = int(start_text) + end = int(end_text) + step = int(step_text or "1") + if step == 0: + raise ValueError(f"Slurm range step must be positive: {component!r}") + direction = 1 if end >= start else -1 + width = max(len(start_text), len(end_text)) + stop = end + direction + expanded.extend(f"{number:0{width}d}" for number in range(start, stop, direction * step)) + return expanded + + +def _expand_component(component: str) -> list[str]: + """Recursively expand all bracket expressions in one hostlist component.""" + opening = component.find("[") + if opening < 0: + if "]" in component: + raise ValueError(f"unmatched closing bracket in Slurm nodelist component: {component!r}") + return [component] + + depth = 0 + closing = -1 + for index in range(opening, len(component)): + if component[index] == "[": + depth += 1 + elif component[index] == "]": + depth -= 1 + if depth == 0: + closing = index + break + if closing < 0: + raise ValueError(f"unmatched opening bracket in Slurm nodelist component: {component!r}") + + prefix = component[:opening] + values = _expand_bracket_values(component[opening + 1 : closing]) + suffixes = _expand_component(component[closing + 1 :]) + return [f"{prefix}{value}{suffix}" for value in values for suffix in suffixes] + + +def expand_slurm_nodelist(nodelist: str) -> list[str]: + """Expand a Slurm nodelist without relying on ``scontrol`` in the container.""" + if not nodelist: + raise ValueError("Slurm nodelist must not be empty") + hosts: list[str] = [] + for component in _split_top_level(nodelist): + hosts.extend(_expand_component(component)) + return hosts + + +def parse_worker_node_counts(value: str) -> list[int]: + """Parse comma-separated physical node counts for logical workers.""" + try: + counts = [int(component) for component in value.split(",")] + except ValueError as error: + raise ValueError(f"worker node counts must be comma-separated integers: {value!r}") from error + if not counts or any(count < 1 for count in counts): + raise ValueError(f"every logical worker must use at least one node: {value!r}") + return counts + + +def build_worker_metrics_urls( + nodelist: str, + worker_node_counts: list[int], + infra_node_count: int, + system_port_base: int, +) -> list[str]: + """Map logical worker leaders to upstream srt-slurm system ports.""" + if infra_node_count < 0: + raise ValueError("infra node count must not be negative") + if system_port_base < 1 or system_port_base > 65535: + raise ValueError(f"invalid system port base: {system_port_base}") + + allocated_hosts = expand_slurm_nodelist(nodelist) + expected_nodes = infra_node_count + sum(worker_node_counts) + if len(allocated_hosts) != expected_nodes: + raise ValueError( + "Slurm allocation does not match the custom benchmark topology: " + f"expanded {len(allocated_hosts)} hosts, expected {expected_nodes} " + f"({infra_node_count} infra + {sum(worker_node_counts)} worker nodes)" + ) + + worker_hosts = allocated_hosts[infra_node_count:] + urls: list[str] = [] + process_offset = 0 + for node_count in worker_node_counts: + port = system_port_base + process_offset + if port > 65535: + raise ValueError(f"derived system port exceeds 65535: {port}") + urls.append(f"http://{worker_hosts[process_offset]}:{port}/metrics") + process_offset += node_count + return urls + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--nodelist", required=True) + parser.add_argument("--worker-node-counts", required=True) + parser.add_argument("--infra-node-count", required=True, type=int) + parser.add_argument("--system-port-base", required=True, type=int) + args = parser.parse_args() + + urls = build_worker_metrics_urls( + nodelist=args.nodelist, + worker_node_counts=parse_worker_node_counts(args.worker_node_counts), + infra_node_count=args.infra_node_count, + system_port_base=args.system_port_base, + ) + print(",".join(urls)) + + +if __name__ == "__main__": + main() diff --git a/utils/agentic/test_srt_metrics_endpoints.py b/utils/agentic/test_srt_metrics_endpoints.py new file mode 100644 index 0000000000..a7b77aa9fa --- /dev/null +++ b/utils/agentic/test_srt_metrics_endpoints.py @@ -0,0 +1,91 @@ +"""Tests for upstream-only srt-slurm custom benchmark metrics discovery.""" + +from pathlib import Path + +import pytest +import yaml + +from utils.agentic.srt_metrics_endpoints import ( + build_worker_metrics_urls, + expand_slurm_nodelist, + parse_worker_node_counts, +) + + +def test_expand_slurm_nodelist_preserves_order_and_zero_padding() -> None: + assert expand_slurm_nodelist("gb300-[001-003,007],login9") == [ + "gb300-001", + "gb300-002", + "gb300-003", + "gb300-007", + "login9", + ] + + +def test_expand_slurm_nodelist_supports_multiple_bracket_groups_and_steps() -> None: + assert expand_slurm_nodelist("rack[1-2]-node[02-04:2]") == [ + "rack1-node02", + "rack1-node04", + "rack2-node02", + "rack2-node04", + ] + + +def test_build_worker_metrics_urls_uses_logical_leaders_and_process_offsets() -> None: + assert build_worker_metrics_urls( + nodelist="gb300-[00-10]", + worker_node_counts=[2, 2, 2, 4], + infra_node_count=1, + system_port_base=7500, + ) == [ + "http://gb300-01:7500/metrics", + "http://gb300-03:7502/metrics", + "http://gb300-05:7504/metrics", + "http://gb300-07:7506/metrics", + ] + + +def test_build_worker_metrics_urls_rejects_allocation_mismatch() -> None: + with pytest.raises(ValueError, match="expanded 4 hosts, expected 5"): + build_worker_metrics_urls( + nodelist="gb300-[00-03]", + worker_node_counts=[2, 2], + infra_node_count=1, + system_port_base=7500, + ) + + +def test_agentic_recipes_describe_their_upstream_worker_topology() -> None: + recipe_directory = ( + Path(__file__).parents[2] + / "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" + ) + recipe_paths = list(recipe_directory.glob("*.yaml")) + assert recipe_paths + for recipe_path in recipe_paths: + recipe = yaml.safe_load(recipe_path.read_text()) + resources = recipe["resources"] + env = recipe["benchmark"]["env"] + actual_counts = parse_worker_node_counts(env["AIPERF_SRT_WORKER_NODE_COUNTS"]) + + if resources.get("agg_workers"): + assert resources["agg_nodes"] % resources["agg_workers"] == 0 + nodes_per_worker = resources["agg_nodes"] // resources["agg_workers"] + expected_counts = [nodes_per_worker] * resources["agg_workers"] + else: + assert resources["prefill_nodes"] % resources["prefill_workers"] == 0 + assert resources["decode_nodes"] % resources["decode_workers"] == 0 + prefill_nodes_per_worker = resources["prefill_nodes"] // resources["prefill_workers"] + decode_nodes_per_worker = resources["decode_nodes"] // resources["decode_workers"] + expected_counts = [prefill_nodes_per_worker] * resources["prefill_workers"] + expected_counts += [decode_nodes_per_worker] * resources["decode_workers"] + + assert actual_counts == expected_counts, recipe_path.name + assert env["AIPERF_SRT_INFRA_NODE_COUNT"] == "1" + assert env["AIPERF_SRT_SYSTEM_PORT_BASE"] == "7500" + + +@pytest.mark.parametrize("value", ["", "2,0,1", "2,nope,1"]) +def test_parse_worker_node_counts_rejects_invalid_values(value: str) -> None: + with pytest.raises(ValueError): + parse_worker_node_counts(value) From dca05755bb34981f3c7770c5cbf5eee6574fc6fc Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 20 Jul 2026 19:47:17 -0500 Subject: [PATCH 2/2] test(agentic): validate upstream worker endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pin the GB300 DeepSeek-V4 AgentX path to NVIDIA/srt-slurm PR #276 through the upstream pull ref, verify its immutable head, and log whether srt-slurm supplied AIPerf worker metrics endpoints. 中文：将 GB300 DeepSeek-V4 AgentX 路径通过上游 Pull Request 引用固定到 NVIDIA/srt-slurm PR #276，校验不可变的 Head Commit，并在日志中明确标识 AIPerf Worker 指标端点是否由 srt-slurm 提供。 --- benchmarks/multi_node/agentic_srt.sh | 14 +++++++------- runners/launch_gb300-nv.sh | 18 +++++++++++------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh index 5b667c51cc..3681d12352 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/multi_node/agentic_srt.sh @@ -29,13 +29,13 @@ done resolve_trace_source install_agentic_deps -# Upstream srt-slurm passes benchmark.env through to custom commands, but only -# its built-in AIPerf runners discover worker metrics endpoints. Reconstruct -# the same endpoint mapping here from the full job allocation. The recipe -# provides one node count per logical worker in upstream topology order; srt- -# slurm assigns DYN_SYSTEM_PORT sequentially to every physical worker process, -# so each logical leader's port is base + its first physical-process offset. -if [ -z "${AIPERF_SERVER_METRICS_URLS:-}" ] && [ -n "${AIPERF_SRT_WORKER_NODE_COUNTS:-}" ]; then +# NVIDIA/srt-slurm PR #276 supplies authoritative logical-worker endpoints to +# custom benchmarks. Keep the local reconstruction as a fallback for tagged +# upstream releases, and log the selected path so the live validation proves +# which implementation produced the endpoint list. +if [ -n "${AIPERF_SERVER_METRICS_URLS:-}" ]; then + echo "Using AIPerf worker metrics endpoints supplied by srt-slurm: $AIPERF_SERVER_METRICS_URLS" +elif [ -n "${AIPERF_SRT_WORKER_NODE_COUNTS:-}" ]; then check_env_vars AIPERF_SRT_INFRA_NODE_COUNT AIPERF_SRT_SYSTEM_PORT_BASE slurm_job_nodelist="${SLURM_JOB_NODELIST:-${SLURM_NODELIST:-}}" if [ -z "$slurm_job_nodelist" ]; then diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 8978ec38ff..58e5dd96d8 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -139,15 +139,19 @@ SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then - # DSv4 GB300 sglang agentic: NVIDIA/srt-slurm v1.0.10 has the nginx - # client_max_body_size fix (>1 MiB agentic warmup bodies), the - # session-affinity frontend, and the BenchmarkType.CUSTOM / extra_mount - # schema these recipes need. Keep this checkout byte-for-byte upstream; - # agentic_srt.sh derives custom-benchmark metrics endpoints from the Slurm - # allocation and recipe-provided logical worker widths. + # Directly validate NVIDIA/srt-slurm PR #276, which exposes authoritative + # logical worker metrics endpoints to custom benchmarks. Fetch the PR + # through NVIDIA's pull ref and verify its immutable head before checkout; + # no downstream patch or custom-fork clone is applied. + SRT_ENDPOINTS_PR_HEAD="3fe550861fee69d8d3823c9a06aa7c2da2916b82" git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout v1.0.10 + git fetch origin pull/276/head + if [[ "$(git rev-parse FETCH_HEAD)" != "$SRT_ENDPOINTS_PR_HEAD" ]]; then + echo "ERROR: NVIDIA/srt-slurm PR #276 head changed; refusing to test an unreviewed commit" >&2 + exit 1 + fi + git checkout --detach "$SRT_ENDPOINTS_PR_HEAD" mkdir -p recipes/sglang/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ recipes/sglang/deepseek-v4/agentic