From 56cda75185f3c145fa41c230da94bc21858e39c6 Mon Sep 17 00:00:00 2001 From: JordanNanos <16803355+JordanNanos@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:39:03 +0000 Subject: [PATCH 1/3] fix(matrix): add curated-full-sweep for kimi+deepseek, exclude qwen/trt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The full sweep had no canonical invocation: a bare `full-sweep --config-files configs/nvidia-master.yaml` sweeps everything (including qwen3.5 and TensorRT), and the generator's --model-prefix/--framework/ --single-node/--multi-node filters AND together within one call, so kimi-single-node + deepseek-multi-node cannot be expressed at once. Add a `curated-full-sweep` subcommand that composes two generate_full_sweep passes into EXACTLY the target set: - kimi single-node on framework vllm - DeepSeek (dsr1/dsv4) multi-node on dynamo-vllm (vLLM), dynamo-sglang (SGLang), llmd-vllm (llm-d) - excludes dynamo-trt/trt and every qwen3.5-* config Only trimming filters (--precision/--runner-type/--seq-lens/--step-size/ --min-conc/--max-conc/--max-tp/--max-ep) are overridable so a smoke run can shrink the sweep without widening the target set. Document it in the workflows README and AGENTS.md, and add a guard test that runs the curated sweep over the real nvidia-master.yaml and asserts the exact target set (model-prefix, node type, framework; no qwen, no trt). 中文：修复全量扫描(full sweep)缺少规范调用入口的问题：直接运行 `full-sweep --config-files configs/nvidia-master.yaml` 会扫描全部配置(包括 qwen3.5 和 TensorRT)，而生成器的 --model-prefix/--framework/--single-node/--multi-node 过滤器在单次调用内按 AND 组合，无法同时表达 kimi 单节点 + deepseek 多节点。 新增 `curated-full-sweep` 子命令，将两次 generate_full_sweep 组合为精确的目标集合： - kimi 单节点，框架 vllm - DeepSeek(dsr1/dsv4)多节点，运行于 dynamo-vllm(vLLM)、dynamo-sglang(SGLang)、 llmd-vllm(llm-d) - 排除 dynamo-trt/trt 及所有 qwen3.5-* 配置 仅暴露用于裁剪的过滤器(--precision/--runner-type/--seq-lens 等)，便于冒烟测试缩小 扫描范围而不会扩大目标集合。已在 workflows README 和 AGENTS.md 中补充文档，并新增守卫 测试：对真实的 nvidia-master.yaml 运行 curated 扫描并断言精确目标集合(model-prefix、 节点类型、框架；无 qwen、无 trt)。 Co-Authored-By: Claude Opus 4.7 --- .github/workflows/README.md | 45 +++++- AGENTS.md | 13 ++ utils/matrix_logic/generate_sweep_configs.py | 140 +++++++++++++++++- .../test_generate_sweep_configs.py | 135 +++++++++++++++++ 4 files changed, 329 insertions(+), 4 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index e330a9484c..dcd6c4ff2e 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -3,16 +3,19 @@ In order to test configurations described in `configs`, the primary workflow file used is `.github/workflows/e2e-tests.yml`. As input, this workflow takes in the CLI arguments for the `utils/matrix_logic/generate_sweep_configs.py` script. The usage for this script is shown below: ``` -usage: generate_sweep_configs.py [-h] {full-sweep,test-config} ... +usage: generate_sweep_configs.py [-h] {full-sweep,curated-full-sweep,test-config} ... Generate benchmark configurations from YAML config files positional arguments: - {full-sweep,test-config} + {full-sweep,curated-full-sweep,test-config} Available commands full-sweep Generate full sweep configurations with optional filtering by model, precision, framework, runner type, and sequence lengths + curated-full-sweep Generate the canonical production full sweep (kimi + single-node vLLM + DeepSeek multi-node vLLM/SGLang/llm-d, + excluding TensorRT and all qwen3.5 configs). test-config Generate full sweep for specific config keys. Supports wildcard patterns (* and ?) for matching multiple keys at once. @@ -90,6 +93,44 @@ full-sweep --multi-node --config-files configs/nvidia-master.yaml full-sweep --scenario-type agentic-coding --config-files configs/nvidia-master.yaml configs/amd-master.yaml ``` +## `curated-full-sweep` Command + +`curated-full-sweep` is the **canonical production full sweep**. It exists because a full sweep must run EXACTLY one target set, and the `full-sweep` filters combine with AND within a single invocation — so "kimi single-node **and** DeepSeek multi-node" cannot be expressed in one `full-sweep` call, and the bare `full-sweep --config-files configs/nvidia-master.yaml` default sweeps *everything* (including qwen3.5 and TensorRT). `curated-full-sweep` composes the two required passes internally and emits exactly: + +- **Kimi single-node** (`model-prefix kimik2.5`) on framework `vllm`. +- **DeepSeek multi-node** (`model-prefix dsr1` / `dsv4`) on the three requested engines, mapped to the concrete framework values in `configs/nvidia-master.yaml`: + - vLLM → `dynamo-vllm` + - SGLang → `dynamo-sglang` + - llm-d → `llmd-vllm` +- **Excluded:** TensorRT (`dynamo-trt` / `trt`) and **every** `qwen3.5-*` config. + +Model prefix, framework, and node-type are fixed by the curated target set and are not overridable. Only trimming filters are exposed (`--precision`, `--runner-type`, `--seq-lens`, `--step-size`, `--min-conc`, `--max-conc`, `--max-tp`, `--max-ep`) so a smoke run can shrink the sweep without ever widening the target set. Eval flags (`--no-evals`, `--evals-only`, `--all-evals`) behave as in `full-sweep`. + +``` +usage: generate_sweep_configs.py curated-full-sweep + --config-files CONFIG_FILES [CONFIG_FILES ...] + [--runner-config RUNNER_CONFIG] + [--no-evals | --evals-only] [--all-evals] + [--precision PRECISION [PRECISION ...]] + [--runner-type RUNNER_TYPE [RUNNER_TYPE ...]] + [--seq-lens {1k1k,8k1k} [{1k1k,8k1k} ...]] + [--step-size STEP_SIZE] + [--min-conc MIN_CONC] [--max-conc MAX_CONC] + [--max-tp MAX_TP] [--max-ep MAX_EP] +``` + +### Examples + +**Dispatch the canonical production full sweep** (pass verbatim as `inputs[generate-cli-command]` to `e2e-tests.yml`): +``` +curated-full-sweep --config-files configs/nvidia-master.yaml +``` + +**Trim to an 8k1k smoke slice for faster validation:** +``` +curated-full-sweep --config-files configs/nvidia-master.yaml --seq-lens 8k1k --max-conc 64 +``` + ## `test-config` Command The `test-config` command generates the full sweep for one or more specific config keys. This is useful for testing individual configurations without filtering by model prefix, framework, etc. diff --git a/AGENTS.md b/AGENTS.md index fdf2765b27..a91f4c9e00 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -64,6 +64,19 @@ python utils/matrix_logic/generate_sweep_configs.py full-sweep \ [--runner-type b200|h100|h200|gb200|...] ``` +The canonical **production full sweep** is `curated-full-sweep` — it runs EXACTLY +kimi single-node on `vllm` plus DeepSeek (dsr1/dsv4) multi-node on `dynamo-vllm` +(vLLM), `dynamo-sglang` (SGLang), and `llmd-vllm` (llm-d), excluding `dynamo-trt` +and every `qwen3.5-*` config. Prefer it over hand-rolled `full-sweep` filters for +full sweeps (a bare `full-sweep --config-files configs/nvidia-master.yaml` sweeps +everything, including qwen and TensorRT). Only trimming filters are overridable: + +```bash +python utils/matrix_logic/generate_sweep_configs.py curated-full-sweep \ + --config-files configs/nvidia-master.yaml \ + [--seq-lens 1k1k|8k1k] [--max-conc N] [--precision fp4|fp8] [--runner-type ...] +``` + Process results: `python utils/process_result.py`. ## Supported Configuration Values diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index e14c50c66b..ee4a2fd7c5 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -1,3 +1,4 @@ +import copy import fnmatch import json import argparse @@ -776,6 +777,72 @@ def generate_full_sweep(args, all_config_data, runner_data): return matrix_values +# Curated "full sweep" target set. A production full sweep must run EXACTLY: +# - Kimi single-node on vLLM (framework `vllm`) +# - DeepSeek (dsr1/dsv4) multi-node on vLLM/SGLang/llm-d, mapped to the +# concrete framework values that exist in configs/nvidia-master.yaml: +# dynamo-vllm (vLLM), dynamo-sglang (SGLang), llmd-vllm (llm-d). +# TensorRT (`dynamo-trt`/`trt`) and every `qwen3.5-*` config are excluded. +# The generator's --model-prefix/--framework/--single-node/--multi-node filters +# combine with AND within a single invocation, so kimi-single-node and +# deepseek-multi-node cannot be expressed at once; the curated sweep composes +# two passes here so the harness has a single entrypoint. +CURATED_SINGLE_NODE_PASS = { + "model-prefix": ["kimik2.5"], + "framework": ["vllm"], +} +CURATED_MULTI_NODE_PASS = { + "model-prefix": ["dsr1", "dsv4"], + "framework": ["dynamo-vllm", "dynamo-sglang", "llmd-vllm"], +} + + +def _curated_pass_args(base_args, *, model_prefix, framework, single_node, multi_node): + """Clone base args, pinning the dimensions the curated sweep fixes.""" + passed = copy.copy(base_args) + passed.model_prefix = model_prefix + passed.framework = framework + passed.single_node = single_node + passed.multi_node = multi_node + return passed + + +def generate_curated_full_sweep(args, all_config_data, runner_data): + """Generate the canonical production full sweep. + + Composes two `generate_full_sweep` passes so the result is EXACTLY the + curated target set (see CURATED_SINGLE_NODE_PASS / CURATED_MULTI_NODE_PASS): + kimi single-node on vLLM plus DeepSeek multi-node on vLLM/SGLang/llm-d, with + TensorRT and all qwen3.5 configs excluded. Optional filters on `args` + (--seq-lens, --min-conc/--max-conc, --step-size, --runner-type, etc.) apply + to both passes, so trimming for a smoke run stays possible without widening + the target set. + """ + single_node = generate_full_sweep( + _curated_pass_args( + args, + model_prefix=CURATED_SINGLE_NODE_PASS["model-prefix"], + framework=CURATED_SINGLE_NODE_PASS["framework"], + single_node=True, + multi_node=False, + ), + all_config_data, + runner_data, + ) + multi_node = generate_full_sweep( + _curated_pass_args( + args, + model_prefix=CURATED_MULTI_NODE_PASS["model-prefix"], + framework=CURATED_MULTI_NODE_PASS["framework"], + single_node=False, + multi_node=True, + ), + all_config_data, + runner_data, + ) + return single_node + multi_node + + def _runner_values_for_filter(runner: str, runner_data: dict, runner_node_filter: str | None) -> list[str]: if not runner_node_filter: return [runner] @@ -1256,6 +1323,73 @@ def main(): help='Show this help message and exit' ) + # Subcommand: curated-full-sweep + curated_parser = subparsers.add_parser( + 'curated-full-sweep', + parents=[parent_parser], + add_help=False, + help=( + 'Generate the canonical production full sweep: kimi single-node on ' + 'vLLM plus DeepSeek multi-node on vLLM/SGLang/llm-d, excluding ' + 'TensorRT and all qwen3.5 configs. Model/framework/node-type are ' + 'fixed by the curated target set; only trimming filters are exposed.' + ) + ) + curated_parser.add_argument( + '--precision', + nargs='+', + required=False, + help='Precision(s) to filter by (e.g., fp4, fp8) (optional, can specify multiple)' + ) + curated_parser.add_argument( + '--runner-type', + nargs='+', + required=False, + help='Runner type(s) to filter by (e.g., h200, gb200) (optional, can specify multiple)' + ) + curated_parser.add_argument( + '--seq-lens', + nargs='+', + choices=list(seq_len_stoi.keys()), + required=False, + help=f"Sequence length configurations to include: {', '.join(seq_len_stoi.keys())}. If not specified, all sequence lengths are included." + ) + curated_parser.add_argument( + '--step-size', + type=int, + default=2, + help='Step size for concurrency values (default: 2)' + ) + curated_parser.add_argument( + '--min-conc', + type=int, + required=False, + help='Minimum concurrency value to include (filters out lower concurrency values)' + ) + curated_parser.add_argument( + '--max-conc', + type=int, + required=False, + help='Maximum concurrency value to include (filters out higher concurrency values)' + ) + curated_parser.add_argument( + '--max-tp', + type=int, + required=False, + help='Maximum tensor parallelism value to include (single-node only)' + ) + curated_parser.add_argument( + '--max-ep', + type=int, + required=False, + help='Maximum expert parallelism value to include (single-node only)' + ) + curated_parser.add_argument( + '-h', '--help', + action='help', + help='Show this help message and exit' + ) + # Subcommand: test-config test_config_keys_parser = subparsers.add_parser( 'test-config', @@ -1291,10 +1425,10 @@ def main(): args = parser.parse_args() apply_node_type_defaults(args) - if args.command == 'full-sweep' and args.step_size <= 1: + if args.command in ('full-sweep', 'curated-full-sweep') and args.step_size <= 1: parser.error("--step-size must be greater than 1") if ( - args.command == 'full-sweep' + args.command in ('full-sweep', 'curated-full-sweep') and args.min_conc is not None and args.max_conc is not None and args.min_conc > args.max_conc @@ -1310,6 +1444,8 @@ def main(): # Route to appropriate function based on subcommand if args.command == 'full-sweep': matrix_values = generate_full_sweep(args, all_config_data, runner_data) + elif args.command == 'curated-full-sweep': + matrix_values = generate_curated_full_sweep(args, all_config_data, runner_data) elif args.command == 'test-config': matrix_values = generate_test_config_sweep(args, all_config_data, runner_data) else: diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 9a1c67d86e..5458541332 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2695,3 +2695,138 @@ def test_prefill_entries_never_in_single_or_evals(self, mixed_entries): assert all('prefill' in x for x in multi) assert all('prefill' not in x for x in single) assert all('prefill' not in x for x in evals) + + +# ============================================================================= +# Guard test: curated production full sweep target set +# ============================================================================= + +import argparse as _argparse +from pathlib import Path as _Path + +from generate_sweep_configs import ( + CURATED_SINGLE_NODE_PASS, + CURATED_MULTI_NODE_PASS, + generate_curated_full_sweep, +) +from validation import load_config_files, load_runner_file + +_REPO_ROOT = _Path(__file__).resolve().parents[2] +_NVIDIA_MASTER = _REPO_ROOT / "configs" / "nvidia-master.yaml" +_RUNNERS = _REPO_ROOT / "configs" / "runners.yaml" + +# Independently-encoded expectation (NOT imported from the generator) so the +# guard cross-checks intent rather than echoing the implementation constants. +_ALLOWED_MODEL_PREFIXES = {"kimik2.5", "dsr1", "dsv4"} +_ALLOWED_FRAMEWORKS = {"vllm", "dynamo-vllm", "dynamo-sglang", "llmd-vllm"} +_FORBIDDEN_FRAMEWORKS = {"dynamo-trt", "trt"} + + +def _curated_base_args(): + """Args mirroring the `curated-full-sweep` subcommand with no trimming.""" + args = _argparse.Namespace() + args.precision = None + args.runner_type = None + args.seq_lens = None + args.step_size = 2 + args.min_conc = None + args.max_conc = None + args.max_tp = None + args.max_ep = None + args.runner_node_filter = None + args.scenario_type = None + return args + + +@pytest.fixture(scope="module") +def curated_entries(): + all_config_data = load_config_files([str(_NVIDIA_MASTER)]) + runner_data = load_runner_file(str(_RUNNERS)) + return generate_curated_full_sweep( + _curated_base_args(), all_config_data, runner_data + ) + + +class TestCuratedFullSweepTargetSet: + """Guard the canonical production full sweep against the real master config. + + Runs `curated-full-sweep` over configs/nvidia-master.yaml and asserts the + generated set is EXACTLY kimi single-node on vLLM plus DeepSeek multi-node + on vLLM/SGLang/llm-d, with TensorRT and all qwen3.5 configs excluded. + """ + + def test_generates_a_non_empty_sweep(self, curated_entries): + assert curated_entries, "curated full sweep unexpectedly empty" + + def test_every_model_prefix_is_in_target_set(self, curated_entries): + prefixes = {e["model-prefix"] for e in curated_entries} + assert prefixes <= _ALLOWED_MODEL_PREFIXES, ( + f"unexpected model-prefix(es): {prefixes - _ALLOWED_MODEL_PREFIXES}" + ) + + def test_no_qwen_entries(self, curated_entries): + assert not any( + e["model-prefix"] == "qwen3.5" or str(e["model-prefix"]).startswith("qwen") + for e in curated_entries + ), "qwen configs must be fully excluded from the curated full sweep" + + def test_every_framework_is_in_target_set(self, curated_entries): + frameworks = {e["framework"] for e in curated_entries} + assert frameworks <= _ALLOWED_FRAMEWORKS, ( + f"unexpected framework(s): {frameworks - _ALLOWED_FRAMEWORKS}" + ) + + def test_no_tensorrt_frameworks(self, curated_entries): + assert not any( + e["framework"] in _FORBIDDEN_FRAMEWORKS for e in curated_entries + ), "dynamo-trt/trt must be excluded from the curated full sweep" + + def test_kimi_entries_are_single_node_vllm(self, curated_entries): + kimi = [e for e in curated_entries if e["model-prefix"] == "kimik2.5"] + assert kimi, "expected kimi single-node entries in the curated sweep" + for e in kimi: + assert "prefill" not in e, "kimi entries must be single-node" + assert e["framework"] == "vllm", "kimi must run on framework vllm" + + def test_deepseek_entries_are_multi_node(self, curated_entries): + deepseek = [ + e for e in curated_entries if e["model-prefix"] in {"dsr1", "dsv4"} + ] + assert deepseek, "expected DeepSeek multi-node entries in the curated sweep" + for e in deepseek: + assert "prefill" in e, "DeepSeek entries must be multi-node" + assert e["framework"] in { + "dynamo-vllm", + "dynamo-sglang", + "llmd-vllm", + } + + def test_all_three_deepseek_engines_are_represented(self, curated_entries): + ds_frameworks = { + e["framework"] + for e in curated_entries + if e["model-prefix"] in {"dsr1", "dsv4"} + } + assert {"dynamo-vllm", "dynamo-sglang", "llmd-vllm"} <= ds_frameworks, ( + f"missing DeepSeek engine(s): " + f"{ {'dynamo-vllm', 'dynamo-sglang', 'llmd-vllm'} - ds_frameworks }" + ) + + def test_master_config_actually_contains_excluded_configs(self): + """Sanity: the exclusions are meaningful because qwen and dynamo-trt + configs really exist in the master config the curated sweep filters.""" + all_config_data = load_config_files([str(_NVIDIA_MASTER)]) + keys = list(all_config_data.keys()) + assert any(k.startswith("qwen3.5") for k in keys), "expected qwen3.5 configs to exist" + assert any("dynamo-trt" in k for k in keys), "expected dynamo-trt configs to exist" + + def test_curated_constants_match_intent(self): + """The generator constants must encode the documented target set.""" + assert CURATED_SINGLE_NODE_PASS == { + "model-prefix": ["kimik2.5"], + "framework": ["vllm"], + } + assert CURATED_MULTI_NODE_PASS == { + "model-prefix": ["dsr1", "dsv4"], + "framework": ["dynamo-vllm", "dynamo-sglang", "llmd-vllm"], + } From 6fb8b39bee8692b53b7cbcc381bdd1c75b7f9d93 Mon Sep 17 00:00:00 2001 From: JordanNanos <16803355+JordanNanos@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:49:27 +0000 Subject: [PATCH 2/3] fix(matrix): rescope curated-full-sweep to exact dashboard chart runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rescope curated-full-sweep to the union of exactly the three ClusterMAX dashboard chart scenarios (the source of truth) and nothing else: - kimik2.5 single-node on vllm - dsr1 single-node on sglang - dsv4 multi-node on dynamo-vllm + llmd-vllm Remove everything the prior version added beyond the charts: dsr1 multi-node, deepseek dynamo-sglang multi-node, and any kimi multi-node. qwen3.5, trt/dynamo-trt, and dynamo-sglang stay excluded. gptoss120b has no active master config (deprecated only) so it is left out of the generated set, documented in a code comment and the docs. Update the guard test to assert this exact set and refresh AGENTS.md and .github/workflows/README.md to match. 中文：将 curated-full-sweep 的范围重新限定为 ClusterMAX 仪表盘图表（作为 唯一事实来源）所渲染的三个场景的并集，别无其他：kimik2.5 单节点 vllm、 dsr1 单节点 sglang、dsv4 多节点 dynamo-vllm 与 llmd-vllm。移除上一版本 中超出图表范围的内容：dsr1 多节点、deepseek 的 dynamo-sglang 多节点以及 任何 kimi 多节点。继续排除 qwen3.5、trt/dynamo-trt 和 dynamo-sglang。 gptoss120b 没有有效的 master 配置（仅存在于 deprecated 目录），因此从 生成集合中剔除，并在代码注释与文档中说明。同步更新守卫测试、AGENTS.md 和 .github/workflows/README.md。 Co-Authored-By: Claude Opus 4.7 --- .github/workflows/README.md | 21 ++-- AGENTS.md | 18 ++- utils/matrix_logic/generate_sweep_configs.py | 114 +++++++++--------- .../test_generate_sweep_configs.py | 97 ++++++++------- 4 files changed, 138 insertions(+), 112 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index dcd6c4ff2e..4538665b20 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -13,9 +13,11 @@ positional arguments: full-sweep Generate full sweep configurations with optional filtering by model, precision, framework, runner type, and sequence lengths - curated-full-sweep Generate the canonical production full sweep (kimi - single-node vLLM + DeepSeek multi-node vLLM/SGLang/llm-d, - excluding TensorRT and all qwen3.5 configs). + curated-full-sweep Generate the canonical production full sweep matching + the ClusterMAX dashboard charts (kimik2.5 single-node + vLLM + dsr1 single-node SGLang + dsv4 multi-node + dynamo-vllm/llmd-vllm), excluding TensorRT, dynamo-sglang + and all qwen3.5 configs. test-config Generate full sweep for specific config keys. Supports wildcard patterns (* and ?) for matching multiple keys at once. @@ -95,14 +97,13 @@ full-sweep --scenario-type agentic-coding --config-files configs/nvidia-master.y ## `curated-full-sweep` Command -`curated-full-sweep` is the **canonical production full sweep**. It exists because a full sweep must run EXACTLY one target set, and the `full-sweep` filters combine with AND within a single invocation — so "kimi single-node **and** DeepSeek multi-node" cannot be expressed in one `full-sweep` call, and the bare `full-sweep --config-files configs/nvidia-master.yaml` default sweeps *everything* (including qwen3.5 and TensorRT). `curated-full-sweep` composes the two required passes internally and emits exactly: +`curated-full-sweep` is the **canonical production full sweep**. It exists because a full sweep must run EXACTLY the model×engine×node-type combinations that the ClusterMAX dashboard charts render — the charts are the source of truth — and the `full-sweep` filters combine with AND within a single invocation, so these three chart scenarios cannot be expressed in one `full-sweep` call (and the bare `full-sweep --config-files configs/nvidia-master.yaml` default sweeps *everything*, including qwen3.5 and TensorRT). `curated-full-sweep` composes the union of exactly three passes internally: -- **Kimi single-node** (`model-prefix kimik2.5`) on framework `vllm`. -- **DeepSeek multi-node** (`model-prefix dsr1` / `dsv4`) on the three requested engines, mapped to the concrete framework values in `configs/nvidia-master.yaml`: - - vLLM → `dynamo-vllm` - - SGLang → `dynamo-sglang` - - llm-d → `llmd-vllm` -- **Excluded:** TensorRT (`dynamo-trt` / `trt`) and **every** `qwen3.5-*` config. +- **kimik2.5 single-node** (`--single-node --model-prefix kimik2.5 --framework vllm`). +- **dsr1 single-node** (`--single-node --model-prefix dsr1 --framework sglang`) — only SGLang single-node dsr1 configs exist; there is no dsr1 single-node vLLM config, and TensorRT is not a chart engine. +- **dsv4 multi-node** (`--multi-node --model-prefix dsv4 --framework dynamo-vllm llmd-vllm`) — the chart's dsv4 multi-node is vLLM-only; `llmd-vllm` emits vllm-prefixed metrics so it renders as vLLM. `dynamo-sglang` and `dynamo-trt` are excluded. +- **Excluded:** TensorRT (`dynamo-trt` / `trt`), `dynamo-sglang`, and **every** `qwen3.5-*` config. +- **gptoss120b:** the chart keeps gptoss120b, but there is **no active master config** for it (only `configs/deprecated/`), so the harness cannot sweep it from the current `nvidia-master.yaml`. It is intentionally left out of the generated set; the chart is fed from other/legacy data, which is out of scope for the harness. Model prefix, framework, and node-type are fixed by the curated target set and are not overridable. Only trimming filters are exposed (`--precision`, `--runner-type`, `--seq-lens`, `--step-size`, `--min-conc`, `--max-conc`, `--max-tp`, `--max-ep`) so a smoke run can shrink the sweep without ever widening the target set. Eval flags (`--no-evals`, `--evals-only`, `--all-evals`) behave as in `full-sweep`. diff --git a/AGENTS.md b/AGENTS.md index a91f4c9e00..51f7ff529a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -64,12 +64,18 @@ python utils/matrix_logic/generate_sweep_configs.py full-sweep \ [--runner-type b200|h100|h200|gb200|...] ``` -The canonical **production full sweep** is `curated-full-sweep` — it runs EXACTLY -kimi single-node on `vllm` plus DeepSeek (dsr1/dsv4) multi-node on `dynamo-vllm` -(vLLM), `dynamo-sglang` (SGLang), and `llmd-vllm` (llm-d), excluding `dynamo-trt` -and every `qwen3.5-*` config. Prefer it over hand-rolled `full-sweep` filters for -full sweeps (a bare `full-sweep --config-files configs/nvidia-master.yaml` sweeps -everything, including qwen and TensorRT). Only trimming filters are overridable: +The canonical **production full sweep** is `curated-full-sweep` — it matches the +ClusterMAX dashboard charts (the source of truth) and runs EXACTLY the union of +three passes: (1) `kimik2.5` single-node on `vllm`, (2) `dsr1` single-node on +`sglang` (no dsr1 single-node vllm config exists; `trt` is not a chart engine), +and (3) `dsv4` multi-node on `dynamo-vllm` and `llmd-vllm` (the chart's dsv4 +multi-node is vLLM-only; `llmd-vllm` emits vllm-prefixed metrics). It excludes +`dynamo-sglang`, `dynamo-trt`/`trt`, and every `qwen3.5-*` config. `gptoss120b` +is a chart scenario but has NO active master config (only `configs/deprecated/`), +so it cannot be swept from `nvidia-master.yaml` and is intentionally left out. +Prefer this over hand-rolled `full-sweep` filters for full sweeps (a bare +`full-sweep --config-files configs/nvidia-master.yaml` sweeps everything, +including qwen and TensorRT). Only trimming filters are overridable: ```bash python utils/matrix_logic/generate_sweep_configs.py curated-full-sweep \ diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index ee4a2fd7c5..0aeba73961 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -777,70 +777,72 @@ def generate_full_sweep(args, all_config_data, runner_data): return matrix_values -# Curated "full sweep" target set. A production full sweep must run EXACTLY: -# - Kimi single-node on vLLM (framework `vllm`) -# - DeepSeek (dsr1/dsv4) multi-node on vLLM/SGLang/llm-d, mapped to the -# concrete framework values that exist in configs/nvidia-master.yaml: -# dynamo-vllm (vLLM), dynamo-sglang (SGLang), llmd-vllm (llm-d). -# TensorRT (`dynamo-trt`/`trt`) and every `qwen3.5-*` config are excluded. +# Curated "full sweep" target set. The ClusterMAX dashboard charts are the +# source of truth: the production full sweep must run EXACTLY the +# model x engine x node-type combinations the charts render, and nothing else. +# Mapped to the concrete framework values in configs/nvidia-master.yaml: +# - kimik2.5 single-node -> framework `vllm` +# - dsr1 single-node -> framework `sglang` (no dsr1 single-node vllm config +# exists, and `trt` is not a chart engine, so dsr1-SN is sglang-only) +# - dsv4 multi-node -> frameworks `dynamo-vllm` + `llmd-vllm` (the chart's +# dsv4 multinode is vLLM-only; llmd-vllm emits vllm-prefixed metrics so it +# renders as vLLM). `dynamo-sglang` and `dynamo-trt` are EXCLUDED. +# Every `qwen3.5-*` config and all TensorRT (`trt`/`dynamo-trt`) are excluded. +# +# gptoss120b is a chart scenario too, but it has NO active master config (it +# lives only under configs/deprecated/), so the harness cannot sweep it from +# nvidia-master.yaml and it is intentionally left out of the generated set. The +# chart keeps gptoss120b from other/legacy data; that is out of scope here. +# # The generator's --model-prefix/--framework/--single-node/--multi-node filters -# combine with AND within a single invocation, so kimi-single-node and -# deepseek-multi-node cannot be expressed at once; the curated sweep composes -# two passes here so the harness has a single entrypoint. -CURATED_SINGLE_NODE_PASS = { - "model-prefix": ["kimik2.5"], - "framework": ["vllm"], -} -CURATED_MULTI_NODE_PASS = { - "model-prefix": ["dsr1", "dsv4"], - "framework": ["dynamo-vllm", "dynamo-sglang", "llmd-vllm"], -} - - -def _curated_pass_args(base_args, *, model_prefix, framework, single_node, multi_node): +# combine with AND within a single invocation, so each chart scenario needs its +# own pass; the curated sweep composes them here so the harness has a single +# entrypoint. +CURATED_SWEEP_PASSES = [ + {"model-prefix": ["kimik2.5"], "framework": ["vllm"], "single-node": True}, + {"model-prefix": ["dsr1"], "framework": ["sglang"], "single-node": True}, + { + "model-prefix": ["dsv4"], + "framework": ["dynamo-vllm", "llmd-vllm"], + "single-node": False, + }, +] + + +def _curated_pass_args(base_args, *, model_prefix, framework, single_node): """Clone base args, pinning the dimensions the curated sweep fixes.""" passed = copy.copy(base_args) passed.model_prefix = model_prefix passed.framework = framework passed.single_node = single_node - passed.multi_node = multi_node + passed.multi_node = not single_node return passed def generate_curated_full_sweep(args, all_config_data, runner_data): """Generate the canonical production full sweep. - Composes two `generate_full_sweep` passes so the result is EXACTLY the - curated target set (see CURATED_SINGLE_NODE_PASS / CURATED_MULTI_NODE_PASS): - kimi single-node on vLLM plus DeepSeek multi-node on vLLM/SGLang/llm-d, with - TensorRT and all qwen3.5 configs excluded. Optional filters on `args` - (--seq-lens, --min-conc/--max-conc, --step-size, --runner-type, etc.) apply - to both passes, so trimming for a smoke run stays possible without widening - the target set. + Composes the `generate_full_sweep` passes in CURATED_SWEEP_PASSES so the + result is EXACTLY the dashboard chart scenarios: kimi single-node on vLLM, + dsr1 single-node on SGLang, and dsv4 multi-node on dynamo-vllm/llmd-vllm. + qwen3.5, TensorRT, dynamo-sglang, and gptoss (no active master config) are + all excluded. Optional filters on `args` (--seq-lens, --min-conc/--max-conc, + --step-size, --runner-type, etc.) apply to every pass, so trimming for a + smoke run stays possible without widening the target set. """ - single_node = generate_full_sweep( - _curated_pass_args( - args, - model_prefix=CURATED_SINGLE_NODE_PASS["model-prefix"], - framework=CURATED_SINGLE_NODE_PASS["framework"], - single_node=True, - multi_node=False, - ), - all_config_data, - runner_data, - ) - multi_node = generate_full_sweep( - _curated_pass_args( - args, - model_prefix=CURATED_MULTI_NODE_PASS["model-prefix"], - framework=CURATED_MULTI_NODE_PASS["framework"], - single_node=False, - multi_node=True, - ), - all_config_data, - runner_data, - ) - return single_node + multi_node + matrix_values = [] + for sweep_pass in CURATED_SWEEP_PASSES: + matrix_values += generate_full_sweep( + _curated_pass_args( + args, + model_prefix=sweep_pass["model-prefix"], + framework=sweep_pass["framework"], + single_node=sweep_pass["single-node"], + ), + all_config_data, + runner_data, + ) + return matrix_values def _runner_values_for_filter(runner: str, runner_data: dict, runner_node_filter: str | None) -> list[str]: @@ -1329,10 +1331,12 @@ def main(): parents=[parent_parser], add_help=False, help=( - 'Generate the canonical production full sweep: kimi single-node on ' - 'vLLM plus DeepSeek multi-node on vLLM/SGLang/llm-d, excluding ' - 'TensorRT and all qwen3.5 configs. Model/framework/node-type are ' - 'fixed by the curated target set; only trimming filters are exposed.' + 'Generate the canonical production full sweep matching the ' + 'ClusterMAX dashboard charts: kimik2.5 single-node vLLM + dsr1 ' + 'single-node SGLang + dsv4 multi-node dynamo-vllm/llmd-vllm, ' + 'excluding TensorRT, dynamo-sglang and all qwen3.5 configs. ' + 'Model/framework/node-type are fixed by the curated target set; ' + 'only trimming filters are exposed.' ) ) curated_parser.add_argument( diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 5458541332..3655a24744 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2705,8 +2705,7 @@ def test_prefill_entries_never_in_single_or_evals(self, mixed_entries): from pathlib import Path as _Path from generate_sweep_configs import ( - CURATED_SINGLE_NODE_PASS, - CURATED_MULTI_NODE_PASS, + CURATED_SWEEP_PASSES, generate_curated_full_sweep, ) from validation import load_config_files, load_runner_file @@ -2717,9 +2716,16 @@ def test_prefill_entries_never_in_single_or_evals(self, mixed_entries): # Independently-encoded expectation (NOT imported from the generator) so the # guard cross-checks intent rather than echoing the implementation constants. +# The ClusterMAX dashboard charts are the source of truth; the curated sweep +# must render exactly the chart scenarios: +# - kimik2.5 single-node -> vllm +# - dsr1 single-node -> sglang +# - dsv4 multi-node -> dynamo-vllm / llmd-vllm +# gptoss120b is a chart scenario but has no active master config, so it is +# absent from the generated set. _ALLOWED_MODEL_PREFIXES = {"kimik2.5", "dsr1", "dsv4"} -_ALLOWED_FRAMEWORKS = {"vllm", "dynamo-vllm", "dynamo-sglang", "llmd-vllm"} -_FORBIDDEN_FRAMEWORKS = {"dynamo-trt", "trt"} +_ALLOWED_FRAMEWORKS = {"vllm", "sglang", "dynamo-vllm", "llmd-vllm"} +_FORBIDDEN_FRAMEWORKS = {"dynamo-trt", "trt", "dynamo-sglang"} def _curated_base_args(): @@ -2751,8 +2757,9 @@ class TestCuratedFullSweepTargetSet: """Guard the canonical production full sweep against the real master config. Runs `curated-full-sweep` over configs/nvidia-master.yaml and asserts the - generated set is EXACTLY kimi single-node on vLLM plus DeepSeek multi-node - on vLLM/SGLang/llm-d, with TensorRT and all qwen3.5 configs excluded. + generated set is EXACTLY the dashboard chart scenarios: kimi single-node on + vLLM, dsr1 single-node on SGLang, and dsv4 multi-node on dynamo-vllm / + llmd-vllm. qwen3.5, TensorRT, dynamo-sglang, and gptoss are all excluded. """ def test_generates_a_non_empty_sweep(self, curated_entries): @@ -2766,20 +2773,26 @@ def test_every_model_prefix_is_in_target_set(self, curated_entries): def test_no_qwen_entries(self, curated_entries): assert not any( - e["model-prefix"] == "qwen3.5" or str(e["model-prefix"]).startswith("qwen") - for e in curated_entries + str(e["model-prefix"]).startswith("qwen") for e in curated_entries ), "qwen configs must be fully excluded from the curated full sweep" + def test_no_gptoss_entries(self, curated_entries): + # gptoss120b is a chart scenario but has no active master config + # (deprecated-only), so it cannot appear in the generated set. + assert not any( + str(e["model-prefix"]).startswith("gptoss") for e in curated_entries + ), "gptoss has no active master config and must not appear" + def test_every_framework_is_in_target_set(self, curated_entries): frameworks = {e["framework"] for e in curated_entries} assert frameworks <= _ALLOWED_FRAMEWORKS, ( f"unexpected framework(s): {frameworks - _ALLOWED_FRAMEWORKS}" ) - def test_no_tensorrt_frameworks(self, curated_entries): + def test_no_forbidden_frameworks(self, curated_entries): assert not any( e["framework"] in _FORBIDDEN_FRAMEWORKS for e in curated_entries - ), "dynamo-trt/trt must be excluded from the curated full sweep" + ), "trt/dynamo-trt/dynamo-sglang must be excluded from the curated full sweep" def test_kimi_entries_are_single_node_vllm(self, curated_entries): kimi = [e for e in curated_entries if e["model-prefix"] == "kimik2.5"] @@ -2788,45 +2801,47 @@ def test_kimi_entries_are_single_node_vllm(self, curated_entries): assert "prefill" not in e, "kimi entries must be single-node" assert e["framework"] == "vllm", "kimi must run on framework vllm" - def test_deepseek_entries_are_multi_node(self, curated_entries): - deepseek = [ - e for e in curated_entries if e["model-prefix"] in {"dsr1", "dsv4"} - ] - assert deepseek, "expected DeepSeek multi-node entries in the curated sweep" - for e in deepseek: - assert "prefill" in e, "DeepSeek entries must be multi-node" - assert e["framework"] in { - "dynamo-vllm", - "dynamo-sglang", - "llmd-vllm", - } + def test_dsr1_entries_are_single_node_sglang(self, curated_entries): + dsr1 = [e for e in curated_entries if e["model-prefix"] == "dsr1"] + assert dsr1, "expected dsr1 single-node entries in the curated sweep" + for e in dsr1: + assert "prefill" not in e, "dsr1 entries must be single-node" + assert e["framework"] == "sglang", "dsr1 must run on framework sglang" + + def test_dsv4_entries_are_multi_node_vllm_engines(self, curated_entries): + dsv4 = [e for e in curated_entries if e["model-prefix"] == "dsv4"] + assert dsv4, "expected dsv4 multi-node entries in the curated sweep" + for e in dsv4: + assert "prefill" in e, "dsv4 entries must be multi-node" + assert e["framework"] in {"dynamo-vllm", "llmd-vllm"}, ( + "dsv4 multi-node must be vLLM-only (dynamo-vllm / llmd-vllm)" + ) - def test_all_three_deepseek_engines_are_represented(self, curated_entries): - ds_frameworks = { - e["framework"] - for e in curated_entries - if e["model-prefix"] in {"dsr1", "dsv4"} + def test_dsv4_multinode_vllm_engines_are_represented(self, curated_entries): + dsv4_frameworks = { + e["framework"] for e in curated_entries if e["model-prefix"] == "dsv4" } - assert {"dynamo-vllm", "dynamo-sglang", "llmd-vllm"} <= ds_frameworks, ( - f"missing DeepSeek engine(s): " - f"{ {'dynamo-vllm', 'dynamo-sglang', 'llmd-vllm'} - ds_frameworks }" + assert {"dynamo-vllm", "llmd-vllm"} <= dsv4_frameworks, ( + f"missing dsv4 engine(s): {{'dynamo-vllm', 'llmd-vllm'}} - {dsv4_frameworks}" ) def test_master_config_actually_contains_excluded_configs(self): - """Sanity: the exclusions are meaningful because qwen and dynamo-trt - configs really exist in the master config the curated sweep filters.""" + """Sanity: the exclusions are meaningful because qwen, dynamo-trt, and + dynamo-sglang configs really exist in the master config being filtered.""" all_config_data = load_config_files([str(_NVIDIA_MASTER)]) keys = list(all_config_data.keys()) assert any(k.startswith("qwen3.5") for k in keys), "expected qwen3.5 configs to exist" assert any("dynamo-trt" in k for k in keys), "expected dynamo-trt configs to exist" + assert any("dynamo-sglang" in k for k in keys), "expected dynamo-sglang configs to exist" def test_curated_constants_match_intent(self): - """The generator constants must encode the documented target set.""" - assert CURATED_SINGLE_NODE_PASS == { - "model-prefix": ["kimik2.5"], - "framework": ["vllm"], - } - assert CURATED_MULTI_NODE_PASS == { - "model-prefix": ["dsr1", "dsv4"], - "framework": ["dynamo-vllm", "dynamo-sglang", "llmd-vllm"], - } + """The generator passes must encode the documented chart target set.""" + assert CURATED_SWEEP_PASSES == [ + {"model-prefix": ["kimik2.5"], "framework": ["vllm"], "single-node": True}, + {"model-prefix": ["dsr1"], "framework": ["sglang"], "single-node": True}, + { + "model-prefix": ["dsv4"], + "framework": ["dynamo-vllm", "llmd-vllm"], + "single-node": False, + }, + ] From c656f5c758de25760b0286680579b05d932028ef Mon Sep 17 00:00:00 2001 From: JordanNanos <16803355+JordanNanos@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:43:16 +0000 Subject: [PATCH 3/3] fix(matrix): narrow curated-full-sweep to two active dashboard tabs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dashboard now keeps only two active tabs, so rescope curated-full-sweep to the union of exactly two passes: - kimik2.5 single-node on vllm - dsv4 multi-node on dynamo-vllm + llmd-vllm Drop the dsr1 single-node (sglang) pass entirely. dsr1 single-node and gptoss120b single-node are deprecated tabs. This is deselection, not deletion: dsr1 configs stay in nvidia-master.yaml, perf-changelog.yaml is untouched, and no historical data or files are removed. qwen3.5, trt/dynamo-trt, and dynamo-sglang stay excluded. Update the guard test to assert the two-pass set (no dsr1, no gptoss, no qwen, no trt/dynamo-trt/dynamo-sglang) and that the excluded configs — including dsr1 single-node — still EXIST in the master config. Refresh AGENTS.md and .github/workflows/README.md. 中文：将 curated-full-sweep 收窄为仪表盘当前保留的两个活跃标签页对应的两个 pass 的并集：kimik2.5 单节点 vllm，以及 dsv4 多节点 dynamo-vllm 与 llmd-vllm。 完全移除 dsr1 单节点（sglang）pass。dsr1 单节点与 gptoss120b 单节点均为已弃用 标签页。此操作为取消选择而非删除：dsr1 配置仍保留在 nvidia-master.yaml 中， perf-changelog.yaml 未改动，未删除任何历史数据或文件。继续排除 qwen3.5、 trt/dynamo-trt 和 dynamo-sglang。更新守卫测试以断言两 pass 集合（无 dsr1、无 gptoss、无 qwen、无 trt/dynamo-trt/dynamo-sglang），并断言被排除的配置——包括 dsr1 单节点——仍存在于 master 配置中。同步更新 AGENTS.md 与 .github/workflows/README.md。 Co-Authored-By: Claude Opus 4.7 --- .github/workflows/README.md | 9 ++-- AGENTS.md | 18 ++++--- utils/matrix_logic/generate_sweep_configs.py | 38 ++++++------- .../test_generate_sweep_configs.py | 53 ++++++++++++------- 4 files changed, 67 insertions(+), 51 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 4538665b20..1f011eff24 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -15,8 +15,8 @@ positional arguments: and sequence lengths curated-full-sweep Generate the canonical production full sweep matching the ClusterMAX dashboard charts (kimik2.5 single-node - vLLM + dsr1 single-node SGLang + dsv4 multi-node - dynamo-vllm/llmd-vllm), excluding TensorRT, dynamo-sglang + vLLM + dsv4 multi-node dynamo-vllm/llmd-vllm), excluding + TensorRT, dynamo-sglang, dsr1 single-node (deprecated) and all qwen3.5 configs. test-config Generate full sweep for specific config keys. Supports wildcard patterns (* and ?) for matching @@ -97,13 +97,12 @@ full-sweep --scenario-type agentic-coding --config-files configs/nvidia-master.y ## `curated-full-sweep` Command -`curated-full-sweep` is the **canonical production full sweep**. It exists because a full sweep must run EXACTLY the model×engine×node-type combinations that the ClusterMAX dashboard charts render — the charts are the source of truth — and the `full-sweep` filters combine with AND within a single invocation, so these three chart scenarios cannot be expressed in one `full-sweep` call (and the bare `full-sweep --config-files configs/nvidia-master.yaml` default sweeps *everything*, including qwen3.5 and TensorRT). `curated-full-sweep` composes the union of exactly three passes internally: +`curated-full-sweep` is the **canonical production full sweep**. It exists because a full sweep must run EXACTLY the model×engine×node-type combinations that the ClusterMAX dashboard charts render — the charts are the source of truth — and the `full-sweep` filters combine with AND within a single invocation, so these chart scenarios cannot be expressed in one `full-sweep` call (and the bare `full-sweep --config-files configs/nvidia-master.yaml` default sweeps *everything*, including qwen3.5 and TensorRT). The dashboard keeps only two active tabs, so `curated-full-sweep` composes the union of exactly two passes internally: - **kimik2.5 single-node** (`--single-node --model-prefix kimik2.5 --framework vllm`). -- **dsr1 single-node** (`--single-node --model-prefix dsr1 --framework sglang`) — only SGLang single-node dsr1 configs exist; there is no dsr1 single-node vLLM config, and TensorRT is not a chart engine. - **dsv4 multi-node** (`--multi-node --model-prefix dsv4 --framework dynamo-vllm llmd-vllm`) — the chart's dsv4 multi-node is vLLM-only; `llmd-vllm` emits vllm-prefixed metrics so it renders as vLLM. `dynamo-sglang` and `dynamo-trt` are excluded. - **Excluded:** TensorRT (`dynamo-trt` / `trt`), `dynamo-sglang`, and **every** `qwen3.5-*` config. -- **gptoss120b:** the chart keeps gptoss120b, but there is **no active master config** for it (only `configs/deprecated/`), so the harness cannot sweep it from the current `nvidia-master.yaml`. It is intentionally left out of the generated set; the chart is fed from other/legacy data, which is out of scope for the harness. +- **Deprecated / deselected:** `dsr1` single-node (SGLang) and `gptoss120b` single-node are deprecated tabs and are no longer part of the curated sweep. `dsr1` configs remain in `nvidia-master.yaml` (deselected, not deleted — other one-off dispatches may still use them). `gptoss120b` has **no active master config** (only `configs/deprecated/`), so the harness cannot sweep it from the current `nvidia-master.yaml` anyway; its chart is fed from other/legacy data, which is out of scope for the harness. Model prefix, framework, and node-type are fixed by the curated target set and are not overridable. Only trimming filters are exposed (`--precision`, `--runner-type`, `--seq-lens`, `--step-size`, `--min-conc`, `--max-conc`, `--max-tp`, `--max-ep`) so a smoke run can shrink the sweep without ever widening the target set. Eval flags (`--no-evals`, `--evals-only`, `--all-evals`) behave as in `full-sweep`. diff --git a/AGENTS.md b/AGENTS.md index 51f7ff529a..154fbcfa04 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,14 +65,16 @@ python utils/matrix_logic/generate_sweep_configs.py full-sweep \ ``` The canonical **production full sweep** is `curated-full-sweep` — it matches the -ClusterMAX dashboard charts (the source of truth) and runs EXACTLY the union of -three passes: (1) `kimik2.5` single-node on `vllm`, (2) `dsr1` single-node on -`sglang` (no dsr1 single-node vllm config exists; `trt` is not a chart engine), -and (3) `dsv4` multi-node on `dynamo-vllm` and `llmd-vllm` (the chart's dsv4 -multi-node is vLLM-only; `llmd-vllm` emits vllm-prefixed metrics). It excludes -`dynamo-sglang`, `dynamo-trt`/`trt`, and every `qwen3.5-*` config. `gptoss120b` -is a chart scenario but has NO active master config (only `configs/deprecated/`), -so it cannot be swept from `nvidia-master.yaml` and is intentionally left out. +ClusterMAX dashboard charts (the source of truth), which keep only two active +tabs, and runs EXACTLY the union of two passes: (1) `kimik2.5` single-node on +`vllm`, and (2) `dsv4` multi-node on `dynamo-vllm` and `llmd-vllm` (the chart's +dsv4 multi-node is vLLM-only; `llmd-vllm` emits vllm-prefixed metrics). It +excludes `dynamo-sglang`, `dynamo-trt`/`trt`, and every `qwen3.5-*` config. +`dsr1` single-node (SGLang) and `gptoss120b` single-node are **deprecated** tabs +and are no longer part of the curated sweep — `dsr1` configs remain in +`nvidia-master.yaml` (deselected, not deleted; other one-off dispatches may still +use them), and `gptoss120b` has no active master config (only +`configs/deprecated/`) so it cannot be swept from `nvidia-master.yaml` anyway. Prefer this over hand-rolled `full-sweep` filters for full sweeps (a bare `full-sweep --config-files configs/nvidia-master.yaml` sweeps everything, including qwen and TensorRT). Only trimming filters are overridable: diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 0aeba73961..ff9e16a010 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -780,19 +780,20 @@ def generate_full_sweep(args, all_config_data, runner_data): # Curated "full sweep" target set. The ClusterMAX dashboard charts are the # source of truth: the production full sweep must run EXACTLY the # model x engine x node-type combinations the charts render, and nothing else. -# Mapped to the concrete framework values in configs/nvidia-master.yaml: +# The dashboard keeps only two active tabs, so the curated sweep is the union of +# two passes, mapped to the concrete framework values in nvidia-master.yaml: # - kimik2.5 single-node -> framework `vllm` -# - dsr1 single-node -> framework `sglang` (no dsr1 single-node vllm config -# exists, and `trt` is not a chart engine, so dsr1-SN is sglang-only) # - dsv4 multi-node -> frameworks `dynamo-vllm` + `llmd-vllm` (the chart's # dsv4 multinode is vLLM-only; llmd-vllm emits vllm-prefixed metrics so it # renders as vLLM). `dynamo-sglang` and `dynamo-trt` are EXCLUDED. # Every `qwen3.5-*` config and all TensorRT (`trt`/`dynamo-trt`) are excluded. # -# gptoss120b is a chart scenario too, but it has NO active master config (it -# lives only under configs/deprecated/), so the harness cannot sweep it from -# nvidia-master.yaml and it is intentionally left out of the generated set. The -# chart keeps gptoss120b from other/legacy data; that is out of scope here. +# Deprecated / deselected chart scenarios (their configs stay in the master +# config for one-off dispatches, but the curated sweep no longer selects them): +# - dsr1 single-node (SGLang): deprecated tab, removed from the harness sweep. +# - gptoss120b single-node: has NO active master config (lives only under +# configs/deprecated/), so it cannot be swept from nvidia-master.yaml anyway. +# The charts keep these from other/legacy data; that is out of scope here. # # The generator's --model-prefix/--framework/--single-node/--multi-node filters # combine with AND within a single invocation, so each chart scenario needs its @@ -800,7 +801,6 @@ def generate_full_sweep(args, all_config_data, runner_data): # entrypoint. CURATED_SWEEP_PASSES = [ {"model-prefix": ["kimik2.5"], "framework": ["vllm"], "single-node": True}, - {"model-prefix": ["dsr1"], "framework": ["sglang"], "single-node": True}, { "model-prefix": ["dsv4"], "framework": ["dynamo-vllm", "llmd-vllm"], @@ -823,12 +823,12 @@ def generate_curated_full_sweep(args, all_config_data, runner_data): """Generate the canonical production full sweep. Composes the `generate_full_sweep` passes in CURATED_SWEEP_PASSES so the - result is EXACTLY the dashboard chart scenarios: kimi single-node on vLLM, - dsr1 single-node on SGLang, and dsv4 multi-node on dynamo-vllm/llmd-vllm. - qwen3.5, TensorRT, dynamo-sglang, and gptoss (no active master config) are - all excluded. Optional filters on `args` (--seq-lens, --min-conc/--max-conc, - --step-size, --runner-type, etc.) apply to every pass, so trimming for a - smoke run stays possible without widening the target set. + result is EXACTLY the two active dashboard chart scenarios: kimi single-node + on vLLM, and dsv4 multi-node on dynamo-vllm/llmd-vllm. qwen3.5, TensorRT, + dynamo-sglang, dsr1 single-node (deprecated), and gptoss (no active master + config) are all excluded. Optional filters on `args` (--seq-lens, + --min-conc/--max-conc, --step-size, --runner-type, etc.) apply to every pass, + so trimming for a smoke run stays possible without widening the target set. """ matrix_values = [] for sweep_pass in CURATED_SWEEP_PASSES: @@ -1332,11 +1332,11 @@ def main(): add_help=False, help=( 'Generate the canonical production full sweep matching the ' - 'ClusterMAX dashboard charts: kimik2.5 single-node vLLM + dsr1 ' - 'single-node SGLang + dsv4 multi-node dynamo-vllm/llmd-vllm, ' - 'excluding TensorRT, dynamo-sglang and all qwen3.5 configs. ' - 'Model/framework/node-type are fixed by the curated target set; ' - 'only trimming filters are exposed.' + 'ClusterMAX dashboard charts: kimik2.5 single-node vLLM + dsv4 ' + 'multi-node dynamo-vllm/llmd-vllm, excluding TensorRT, ' + 'dynamo-sglang, dsr1 single-node (deprecated) and all qwen3.5 ' + 'configs. Model/framework/node-type are fixed by the curated target ' + 'set; only trimming filters are exposed.' ) ) curated_parser.add_argument( diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 3655a24744..8483c006ca 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2716,15 +2716,14 @@ def test_prefill_entries_never_in_single_or_evals(self, mixed_entries): # Independently-encoded expectation (NOT imported from the generator) so the # guard cross-checks intent rather than echoing the implementation constants. -# The ClusterMAX dashboard charts are the source of truth; the curated sweep -# must render exactly the chart scenarios: +# The ClusterMAX dashboard charts are the source of truth; the dashboard keeps +# only two active tabs, so the curated sweep must render exactly: # - kimik2.5 single-node -> vllm -# - dsr1 single-node -> sglang # - dsv4 multi-node -> dynamo-vllm / llmd-vllm -# gptoss120b is a chart scenario but has no active master config, so it is -# absent from the generated set. -_ALLOWED_MODEL_PREFIXES = {"kimik2.5", "dsr1", "dsv4"} -_ALLOWED_FRAMEWORKS = {"vllm", "sglang", "dynamo-vllm", "llmd-vllm"} +# dsr1 single-node (deprecated) and gptoss120b (no active master config) are +# absent from the generated set; their configs stay in the master config. +_ALLOWED_MODEL_PREFIXES = {"kimik2.5", "dsv4"} +_ALLOWED_FRAMEWORKS = {"vllm", "dynamo-vllm", "llmd-vllm"} _FORBIDDEN_FRAMEWORKS = {"dynamo-trt", "trt", "dynamo-sglang"} @@ -2757,9 +2756,10 @@ class TestCuratedFullSweepTargetSet: """Guard the canonical production full sweep against the real master config. Runs `curated-full-sweep` over configs/nvidia-master.yaml and asserts the - generated set is EXACTLY the dashboard chart scenarios: kimi single-node on - vLLM, dsr1 single-node on SGLang, and dsv4 multi-node on dynamo-vllm / - llmd-vllm. qwen3.5, TensorRT, dynamo-sglang, and gptoss are all excluded. + generated set is EXACTLY the two active dashboard chart scenarios: kimi + single-node on vLLM, and dsv4 multi-node on dynamo-vllm / llmd-vllm. qwen3.5, + TensorRT, dynamo-sglang, dsr1 single-node (deprecated), and gptoss are all + excluded. """ def test_generates_a_non_empty_sweep(self, curated_entries): @@ -2801,12 +2801,12 @@ def test_kimi_entries_are_single_node_vllm(self, curated_entries): assert "prefill" not in e, "kimi entries must be single-node" assert e["framework"] == "vllm", "kimi must run on framework vllm" - def test_dsr1_entries_are_single_node_sglang(self, curated_entries): - dsr1 = [e for e in curated_entries if e["model-prefix"] == "dsr1"] - assert dsr1, "expected dsr1 single-node entries in the curated sweep" - for e in dsr1: - assert "prefill" not in e, "dsr1 entries must be single-node" - assert e["framework"] == "sglang", "dsr1 must run on framework sglang" + def test_no_dsr1_entries(self, curated_entries): + # dsr1 single-node is a deprecated tab; it is deselected from the curated + # sweep (its configs remain in the master config for one-off dispatch). + assert not any( + str(e["model-prefix"]).startswith("dsr1") for e in curated_entries + ), "dsr1 is deprecated and must not appear in the curated full sweep" def test_dsv4_entries_are_multi_node_vllm_engines(self, curated_entries): dsv4 = [e for e in curated_entries if e["model-prefix"] == "dsv4"] @@ -2826,19 +2826,34 @@ def test_dsv4_multinode_vllm_engines_are_represented(self, curated_entries): ) def test_master_config_actually_contains_excluded_configs(self): - """Sanity: the exclusions are meaningful because qwen, dynamo-trt, and - dynamo-sglang configs really exist in the master config being filtered.""" + """Sanity: the exclusions are meaningful because qwen, dynamo-trt, + dynamo-sglang, and dsr1 single-node configs really exist in the master + config being filtered. Their absence from the curated sweep is + deselection, NOT deletion — the entries are preserved in the master + config for one-off dispatches.""" all_config_data = load_config_files([str(_NVIDIA_MASTER)]) keys = list(all_config_data.keys()) assert any(k.startswith("qwen3.5") for k in keys), "expected qwen3.5 configs to exist" assert any("dynamo-trt" in k for k in keys), "expected dynamo-trt configs to exist" assert any("dynamo-sglang" in k for k in keys), "expected dynamo-sglang configs to exist" + # dsr1 single-node (sglang, multinode=False) must still exist in the + # master config even though the curated sweep no longer selects it. + dsr1_single_node = [ + k + for k, v in all_config_data.items() + if k.startswith("dsr1") + and v.get("framework") == "sglang" + and not v.get("multinode", False) + ] + assert dsr1_single_node, ( + "expected dsr1 single-node sglang configs to still exist in master " + "(deselected from curated sweep, not deleted)" + ) def test_curated_constants_match_intent(self): """The generator passes must encode the documented chart target set.""" assert CURATED_SWEEP_PASSES == [ {"model-prefix": ["kimik2.5"], "framework": ["vllm"], "single-node": True}, - {"model-prefix": ["dsr1"], "framework": ["sglang"], "single-node": True}, { "model-prefix": ["dsv4"], "framework": ["dynamo-vllm", "llmd-vllm"],