From 0b159e3bc47504bf66efd0cb5b29f543dac8b83a Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Fri, 12 Jun 2026 12:40:10 +0200 Subject: [PATCH 01/40] chore: move all cli functions into a single location Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/benchmarks.py | 189 +------------- nemo_gym/cli/__init__.py | 14 ++ nemo_gym/cli/dataset.py | 96 +++++++ nemo_gym/cli/dev.py | 37 +++ nemo_gym/{cli.py => cli/env.py} | 211 +--------------- nemo_gym/cli/eval.py | 330 +++++++++++++++++++++++++ nemo_gym/cli/general.py | 147 +++++++++++ nemo_gym/dataset_orchestrator.py | 34 --- nemo_gym/gitlab_utils.py | 12 - nemo_gym/prompt.py | 12 - nemo_gym/reward_profile.py | 29 --- nemo_gym/rollout_collection.py | 15 -- nemo_gym/train_data_utils.py | 12 - pyproject.toml | 100 ++++---- tests/unit_tests/test_benchmarks.py | 34 +-- tests/unit_tests/test_cli.py | 6 +- tests/unit_tests/test_server_status.py | 3 +- 17 files changed, 701 insertions(+), 580 deletions(-) create mode 100644 nemo_gym/cli/__init__.py create mode 100644 nemo_gym/cli/dataset.py create mode 100644 nemo_gym/cli/dev.py rename nemo_gym/{cli.py => cli/env.py} (82%) create mode 100644 nemo_gym/cli/eval.py create mode 100644 nemo_gym/cli/general.py diff --git a/nemo_gym/benchmarks.py b/nemo_gym/benchmarks.py index 9ba1e9fafd..7d57c6a2f6 100644 --- a/nemo_gym/benchmarks.py +++ b/nemo_gym/benchmarks.py @@ -14,26 +14,19 @@ # limitations under the License. """Benchmark discovery and preparation utilities.""" -import importlib -from glob import glob -from multiprocessing import Pool from pathlib import Path -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional -import rich from omegaconf import DictConfig, OmegaConf -from pydantic import BaseModel, Field -from rich.table import Table -from tqdm.auto import tqdm +from pydantic import BaseModel from nemo_gym import PARENT_DIR -from nemo_gym.config_types import BaseNeMoGymCLIConfig, BenchmarkDatasetConfig +from nemo_gym.config_types import BenchmarkDatasetConfig from nemo_gym.global_config import ( POLICY_MODEL_KEY_NAME, GlobalConfigDictParser, GlobalConfigDictParserConfig, get_first_server_config_dict, - get_global_config_dict, ) @@ -105,179 +98,3 @@ def _load_benchmarks_from_config_paths(config_paths: List[Path]) -> Dict[str, Be benchmarks_dict[maybe_bc.name] = maybe_bc return benchmarks_dict - - -def list_benchmarks() -> None: - """CLI command: list available benchmarks.""" - global_config_dict = get_global_config_dict( - global_config_dict_parser_config=GlobalConfigDictParserConfig( - initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, - ) - ) - BaseNeMoGymCLIConfig.model_validate(global_config_dict) - - assert BENCHMARKS_DIR.exists(), "Missing benchmarks directory" - - config_paths = glob("**/config.yaml", root_dir=BENCHMARKS_DIR, recursive=True) - config_paths = [BENCHMARKS_DIR / p for p in config_paths] - config_paths = sorted(config_paths) - - benchmarks = _load_benchmarks_from_config_paths(config_paths) - - if not benchmarks: - rich.print("[yellow]No benchmarks found.[/yellow]") - rich.print(f"Expected benchmarks directory: {BENCHMARKS_DIR}") - return - - table = Table(title=f"Available benchmarks in NeMo Gym ({len(benchmarks)})") - table.add_column("Benchmark name") - table.add_column("Agent name") - table.add_column("Num repeats") - - for name, bench in benchmarks.items(): - table.add_row(name, bench.agent_name, str(bench.num_repeats)) - - rich.print(table) - - -class PrepareBenchmarkConfig(BaseNeMoGymCLIConfig): - """ - Prepare benchmark data by running the benchmark's prepare.py script. - - The benchmark is identified from a config_paths entry pointing to a - benchmarks/*/config.yaml file. - - Examples: - - ```bash - ng_prepare_benchmark "+config_paths=[benchmarks/aime24/config.yaml]" - ``` - """ - - use_cached_prepared_benchmarks: bool = Field( - default=False, description="Skip benchmark preparation if the prepared file is already present" - ) - num_prepare_benchmark_processes: int = Field( - default=1, description="Number of processes to parallelize benchmark preparation" - ) - - -def _multiprocess_benchmark_prepare_fn(args): - benchmark_config: BenchmarkConfig - prepare_module_path: str - (benchmark_config, prepare_module_path) = args - - print(f"Preparing benchmark: {benchmark_config.name}") - - module = importlib.import_module(prepare_module_path) - output_fpath = module.prepare() - assert output_fpath.absolute() == benchmark_config.dataset.jsonl_fpath.absolute(), ( - f"Expected the actual prepared dataset output fpath to match the jsonl_fpath set in the config. Instead got {output_fpath=} jsonl_fpath={benchmark_config.dataset.jsonl_fpath}" - ) - print(f"Benchmark data prepared at: {output_fpath}") - - -def prepare_benchmark() -> None: - """CLI command: prepare benchmark data.""" - global_config_dict = get_global_config_dict( - global_config_dict_parser_config=GlobalConfigDictParserConfig( - initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, - ) - ) - prepare_benchmark_config = PrepareBenchmarkConfig.model_validate(global_config_dict) - - benchmarks_dict: Dict[str, BenchmarkConfig] = dict() - for server_instance_name in global_config_dict: - server_config = global_config_dict[server_instance_name] - if not isinstance(server_config, (dict, DictConfig)) or "responses_api_agents" not in server_config: - continue - - inner_server_config = get_first_server_config_dict(global_config_dict, server_instance_name) - - datasets: List[BenchmarkDatasetConfig] = [] - for dataset in inner_server_config.get("datasets") or []: - if dataset["type"] != "benchmark": - continue - - datasets.append(BenchmarkDatasetConfig.model_validate(dataset)) - - if len(datasets) < 1: - continue - - assert len(datasets) == 1, ( - f"Expected 1 benchmark dataset for `{server_instance_name}`, but found {len(datasets)}!" - ) - - dataset = datasets[0] - - benchmarks_dict[server_instance_name] = BenchmarkConfig( - name=dataset.name, - path=Path(""), - agent_name=server_instance_name, - num_repeats=dataset.num_repeats, - dataset=dataset, - ) - - assert benchmarks_dict, ( - 'No benchmark config found in config_paths. Pass a benchmark config, e.g.: "+config_paths=[benchmarks/aime24/config.yaml]"' - ) - - # Validate all benchmarks before preparing any - prepare_script_missing: List[BenchmarkConfig] = [] - prepare_function_missing: List[BenchmarkConfig] = [] - - validated: List[Tuple[BenchmarkConfig, str]] = [] - already_prepared: List[BenchmarkConfig] = [] - for benchmark_config in benchmarks_dict.values(): - prepare_script_path = benchmark_config.dataset.prepare_script - if not prepare_script_path.exists(): - prepare_script_missing.append(benchmark_config) - continue - - prepare_module_path = ".".join(prepare_script_path.with_suffix("").parts) - module = importlib.import_module(prepare_module_path) - if not hasattr(module, "prepare"): - prepare_function_missing.append(benchmark_config) - continue - - is_already_prepared = benchmark_config.dataset.jsonl_fpath.exists() - if prepare_benchmark_config.use_cached_prepared_benchmarks and is_already_prepared: - already_prepared.append(benchmark_config) - continue - - validated.append((benchmark_config, prepare_module_path)) - - if already_prepared: - already_prepared_str = "".join(f"- {bc.name}: {bc.dataset.jsonl_fpath}\n" for bc in already_prepared) - already_prepared_str = f"""The following benchmarks have already been prepared. Since `use_cached_prepared_benchmarks=true`, we will skip re-preparation of those benchmarks. - {already_prepared_str}""" - print(already_prepared_str) - - errors_to_print = "" - if prepare_script_missing: - prepare_script_missing_str = "".join( - f"- {bc.name}: {bc.dataset.prepare_script}\n" for bc in prepare_script_missing - ) - errors_to_print += f"""The following benchmarks are missing a valid prepare script: -{prepare_script_missing_str} -""" - if prepare_function_missing: # pragma: no cover - prepare_function_missing_str = "".join( - f"- {bc.name}: {bc.dataset.prepare_script}\n" for bc in prepare_function_missing - ) - errors_to_print += f"""The following benchmarks have a prepare script, but are missing the prepare function: -{prepare_function_missing_str} -""" - if errors_to_print: - errors_to_print = f"""Did not prepare any benchmarks due to benchmark config errors. -{errors_to_print}""" - raise RuntimeError(errors_to_print) - - # Prepare after all validations pass - if prepare_benchmark_config.num_prepare_benchmark_processes > 1: # pragma: no cover - with Pool(processes=prepare_benchmark_config.num_prepare_benchmark_processes) as pool: - results = pool.imap_unordered(_multiprocess_benchmark_prepare_fn, validated) - list(tqdm(results, total=len(validated))) - else: - results = map(_multiprocess_benchmark_prepare_fn, validated) - list(tqdm(results, total=len(validated))) diff --git a/nemo_gym/cli/__init__.py b/nemo_gym/cli/__init__.py new file mode 100644 index 0000000000..3159bfe656 --- /dev/null +++ b/nemo_gym/cli/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_gym/cli/dataset.py b/nemo_gym/cli/dataset.py new file mode 100644 index 0000000000..eb3081ed4c --- /dev/null +++ b/nemo_gym/cli/dataset.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_gym.config_types import ( + DeleteJsonlDatasetGitlabConfig, + DownloadJsonlDatasetGitlabConfig, + DownloadJsonlDatasetHuggingFaceConfig, + UploadJsonlDatasetGitlabConfig, + UploadJsonlDatasetHuggingFaceConfig, + UploadJsonlDatasetHuggingFaceMaybeDeleteConfig, +) +from nemo_gym.dataset_orchestrator import ( + delete_jsonl_dataset_from_gitlab, + upload_jsonl_dataset_to_hf_maybe_delete, +) +from nemo_gym.gitlab_utils import download_jsonl_dataset, upload_jsonl_dataset +from nemo_gym.global_config import GlobalConfigDictParserConfig, get_global_config_dict +from nemo_gym.hf_utils import download_hf_dataset_as_jsonl +from nemo_gym.prompt import MaterializePromptsConfig, materialize_prompts +from nemo_gym.train_data_utils import TrainDataProcessor + + +def upload_jsonl_dataset_cli() -> None: # pragma: no cover + global_config = get_global_config_dict() + config = UploadJsonlDatasetGitlabConfig.model_validate(global_config) + upload_jsonl_dataset(config) + + +def download_jsonl_dataset_cli() -> None: # pragma: no cover + global_config = get_global_config_dict() + config = DownloadJsonlDatasetGitlabConfig.model_validate(global_config) + download_jsonl_dataset(config) + + +def upload_jsonl_dataset_to_hf_cli() -> None: # pragma: no cover + global_config = get_global_config_dict() + config = UploadJsonlDatasetHuggingFaceMaybeDeleteConfig.model_validate(global_config) + upload_jsonl_dataset_to_hf_maybe_delete(config, delete_from_gitlab=config.delete_from_gitlab) + + +def download_jsonl_dataset_from_hf_cli() -> None: # pragma: no cover + global_config = get_global_config_dict() + config = DownloadJsonlDatasetHuggingFaceConfig.model_validate(global_config) + + if config.artifact_fpath: + print(f"Downloading file '{config.artifact_fpath}' from '{config.repo_id}'...") + else: + print(f"Downloading '{config.split or 'all'}' split(s) from '{config.repo_id}'...") + + download_hf_dataset_as_jsonl(config) + + +def delete_jsonl_dataset_from_gitlab_cli() -> None: # pragma: no cover + global_config = get_global_config_dict() + config = DeleteJsonlDatasetGitlabConfig.model_validate(global_config) + delete_jsonl_dataset_from_gitlab(config.dataset_name) + + +def upload_jsonl_dataset_to_hf_and_delete_gitlab_cli() -> None: # pragma: no cover + global_config = get_global_config_dict() + config = UploadJsonlDatasetHuggingFaceConfig.model_validate(global_config) + upload_jsonl_dataset_to_hf_maybe_delete(config, delete_from_gitlab=True) + + +def materialize_prompts_cli() -> None: # pragma: no cover + """CLI entry point for ng_materialize_prompts.""" + global_config_dict = get_global_config_dict( + global_config_dict_parser_config=GlobalConfigDictParserConfig( + initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + ) + config = MaterializePromptsConfig.model_validate(global_config_dict) + materialize_prompts(config.input_jsonl_fpath, config.prompt_config, config.output_jsonl_fpath) + + +def prepare_data(): # pragma: no cover + global_config_dict = get_global_config_dict( + global_config_dict_parser_config=GlobalConfigDictParserConfig( + initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + ) + + data_processor = TrainDataProcessor() + data_processor.run(global_config_dict) diff --git a/nemo_gym/cli/dev.py b/nemo_gym/cli/dev.py new file mode 100644 index 0000000000..2134d45d7a --- /dev/null +++ b/nemo_gym/cli/dev.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from subprocess import Popen + +from nemo_gym.config_types import BaseNeMoGymCLIConfig +from nemo_gym.global_config import get_global_config_dict + + +def dev_test(): # pragma: no cover + """ + Run core NeMo Gym tests with coverage reporting (runs pytest with --cov flag). + + Examples: + + ```bash + ng_dev_test + ``` + """ + global_config_dict = get_global_config_dict() + # Just here for help + BaseNeMoGymCLIConfig.model_validate(global_config_dict) + + proc = Popen("pytest --cov=. --durations=10", shell=True) + exit(proc.wait()) diff --git a/nemo_gym/cli.py b/nemo_gym/cli/env.py similarity index 82% rename from nemo_gym/cli.py rename to nemo_gym/cli/env.py index e3df40025a..69c422454b 100644 --- a/nemo_gym/cli.py +++ b/nemo_gym/cli/env.py @@ -12,16 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import asyncio import json import os -import platform import shlex -import sys -from copy import deepcopy from glob import glob -from importlib.metadata import entry_points -from importlib.metadata import version as md_version from os import makedirs from os.path import exists from pathlib import Path @@ -32,16 +28,15 @@ from time import sleep, time from typing import Dict, List, Optional, Tuple -import psutil import rich import uvicorn from devtools import pprint -from omegaconf import DictConfig, OmegaConf, open_dict +from omegaconf import DictConfig, OmegaConf from pydantic import Field from rich.table import Table from tqdm.auto import tqdm -from nemo_gym import PARENT_DIR, ROOT_DIR, __version__ +from nemo_gym import PARENT_DIR, ROOT_DIR from nemo_gym.cli_setup_command import run_command, setup_env_command from nemo_gym.config_types import BaseNeMoGymCLIConfig from nemo_gym.global_config import ( @@ -52,7 +47,6 @@ GlobalConfigDictParserConfig, get_global_config_dict, ) -from nemo_gym.rollout_collection import E2ERolloutCollectionConfig, RolloutCollectionConfig, RolloutCollectionHelper from nemo_gym.server_status import StatusCommand from nemo_gym.server_utils import ( HEAD_SERVER_KEY_NAME, @@ -62,7 +56,6 @@ ServerStatus, initialize_ray, ) -from nemo_gym.train_data_utils import TrainDataProcessor # Grace period after SIGINT before escalating to SIGKILL. Kept short so Ctrl-C is responsive. @@ -428,69 +421,6 @@ def run( rh.run_forever() -def e2e_rollout_collection(): # pragma: no cover - global_config_dict = get_global_config_dict() - - # Ensure we have the right config first thing - e2e_rollout_collection_config = E2ERolloutCollectionConfig.model_validate(global_config_dict) - - # Prepare data - data_processor_config_dict = deepcopy(global_config_dict) - with open_dict(data_processor_config_dict): - data_processor_config_dict["should_download"] = True - data_processor_config_dict["mode"] = "train_preparation" - - output_fpath = Path(e2e_rollout_collection_config.output_jsonl_fpath) - data_process_output_dir = output_fpath.parent / "preprocessed_datasets" - data_processor_config_dict["output_dirpath"] = str(data_process_output_dir) - - input_jsonl_fpath = data_process_output_dir / f"{e2e_rollout_collection_config.split}.jsonl" - should_skip_data_processing = ( - e2e_rollout_collection_config.reuse_existing_data_preparation and input_jsonl_fpath.exists() - ) - if not should_skip_data_processing: - if e2e_rollout_collection_config.reuse_existing_data_preparation: - print( - f"Even though the `reuse_existing_data_preparation=true` flag was set, we will still do data preparation since the final input jsonl fpath `{input_jsonl_fpath}` does not exist yet" - ) - - data_processor = TrainDataProcessor() - data_processor.run(data_processor_config_dict) - else: - print( - f"Skipping data preparation since `reuse_existing_data_preparation=true` and the final input jsonl fpath `{input_jsonl_fpath}` already exists" - ) - - # Convert to RolloutCollectionConfig - rollout_collection_config_dict = deepcopy(global_config_dict) - with open_dict(rollout_collection_config_dict): - assert input_jsonl_fpath.exists(), input_jsonl_fpath - rollout_collection_config_dict["input_jsonl_fpath"] = str(input_jsonl_fpath) - - rollout_collection_config = RolloutCollectionConfig.model_validate( - OmegaConf.to_container(rollout_collection_config_dict) - ) - - rh = RunHelper() - rh.start(None) - - rch = RolloutCollectionHelper() - - print( - f"""Output artifacts: -1. Preprocessed datasets: {data_processor_config_dict["output_dirpath"]} -2. Dataset file used for rollout collection: {rollout_collection_config_dict["input_jsonl_fpath"]} -3. Rollout collection results file: {output_fpath} -""" - ) - try: - asyncio.run(rch.run_from_config(rollout_collection_config)) - except KeyboardInterrupt: - pass - finally: - rh.shutdown() - - def _validate_data_single(test_config: TestConfig) -> None: # pragma: no cover if not test_config.should_validate_data: return @@ -721,24 +651,6 @@ def test_all(): # pragma: no cover exit(1) -def dev_test(): # pragma: no cover - """ - Run core NeMo Gym tests with coverage reporting (runs pytest with --cov flag). - - Examples: - - ```bash - ng_dev_test - ``` - """ - global_config_dict = get_global_config_dict() - # Just here for help - BaseNeMoGymCLIConfig.model_validate(global_config_dict) - - proc = Popen("pytest --cov=. --durations=10", shell=True) - exit(proc.wait()) - - def init_resources_server(): # pragma: no cover """ Initialize a new resources server with template files and directory structure. @@ -887,33 +799,6 @@ def dump_config(): # pragma: no cover print(OmegaConf.to_yaml(global_config_dict, resolve=True)) -def display_help(): - """ - Display a list of available NeMo Gym CLI commands. - - Examples: - - ```bash - ng_help - ``` - """ - global_config_dict = get_global_config_dict() - # Just here for help - BaseNeMoGymCLIConfig.model_validate(global_config_dict) - - eps = entry_points().select(group="console_scripts") - project_scripts = {ep.name: ep.value for ep in eps if ep.name.startswith(("nemo_gym_", "ng_"))} - rich.print("""Run a command with `+h=true` or `+help=true` to see more detailed information! - -[bold]Available CLI scripts[/bold] ------------------""") - for script in project_scripts: - if not script.startswith("ng_"): - continue - - print(script) - - def status(): # pragma: no cover global_config_dict = get_global_config_dict() BaseNeMoGymCLIConfig.model_validate(global_config_dict) @@ -965,93 +850,3 @@ def pip_list(): # pragma: no cover proc = run_command(command, dir_path) return_code = proc.wait() exit(return_code) - - -class VersionConfig(BaseNeMoGymCLIConfig): - """ - Display gym version and system information. - - Examples: - - ```bash - # Display version information - ng_version - - # Output as JSON - ng_version +json=true - ``` - """ - - json_format: bool = Field(default=False, alias="json", description="Output in JSON format for programmatic use.") - - -def version(): # pragma: no cover - """Display gym version and system information.""" - global_config_dict = get_global_config_dict() - config = VersionConfig.model_validate(global_config_dict) - - json_output = config.json_format - - version_info = { - "nemo_gym": __version__, - "python": platform.python_version(), - "python_path": sys.executable, - "installation_path": str(PARENT_DIR), - } - - key_deps = [ - "openai", - "ray", - ] - - dependencies = {dep: md_version(dep) for dep in key_deps} - - version_info["dependencies"] = dependencies - - # System info - version_info["system"] = { - "os": f"{platform.system()} {platform.release()}", - "platform": platform.platform(), - "architecture": platform.machine(), - "processor": platform.processor() or "unknown", - "cpus": os.cpu_count(), - } - - # Memory info - mem = psutil.virtual_memory() - version_info["system"]["memory_gb"] = round(mem.total / (1024**3), 2) - - # Output - if json_output: - print(json.dumps(version_info)) - else: - output = f"""\ -NeMo Gym v{version_info["nemo_gym"]} -Python {version_info["python"]} ({version_info["python_path"]}) -Installation: {version_info["installation_path"]}""" - - if "dependencies" in version_info: - deps_lines = "\n".join(f" {dep}: {ver}" for dep, ver in version_info["dependencies"].items()) - sys_info = version_info["system"] - output += f""" - -Key Dependencies: -{deps_lines} - -System: - OS: {sys_info["os"]} - Platform: {sys_info["platform"]} - Architecture: {sys_info["architecture"]} - Processor: {sys_info["processor"]} - CPUs: {sys_info["cpus"]} - Memory: {sys_info["memory_gb"]} GB""" - - print(output) - - -def reinstall(): # pragma: no cover - global_config_dict = get_global_config_dict() - # Just here for help - BaseNeMoGymCLIConfig.model_validate(global_config_dict) - - Popen("uv sync --extra dev --group docs", shell=True).communicate() diff --git a/nemo_gym/cli/eval.py b/nemo_gym/cli/eval.py new file mode 100644 index 0000000000..54f9b25a84 --- /dev/null +++ b/nemo_gym/cli/eval.py @@ -0,0 +1,330 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import importlib +from copy import deepcopy +from glob import glob +from multiprocessing import Pool +from pathlib import Path +from typing import Dict, List, Tuple + +import orjson +import rich +from omegaconf import DictConfig, OmegaConf, open_dict +from pydantic import Field +from rich.table import Table +from tqdm.auto import tqdm + +from nemo_gym.benchmarks import BENCHMARKS_DIR, BenchmarkConfig, _load_benchmarks_from_config_paths +from nemo_gym.cli.env import RunHelper +from nemo_gym.config_types import BaseNeMoGymCLIConfig, BenchmarkDatasetConfig +from nemo_gym.global_config import ( + ROLLOUT_INDEX_KEY_NAME, + TASK_INDEX_KEY_NAME, + GlobalConfigDictParserConfig, + get_first_server_config_dict, + get_global_config_dict, +) +from nemo_gym.reward_profile import RewardProfileConfig, RewardProfiler +from nemo_gym.rollout_collection import ( + E2ERolloutCollectionConfig, + RolloutAggregationConfig, + RolloutAggregationHelper, + RolloutCollectionConfig, + RolloutCollectionHelper, +) +from nemo_gym.train_data_utils import TrainDataProcessor + + +def list_benchmarks() -> None: + """CLI command: list available benchmarks.""" + global_config_dict = get_global_config_dict( + global_config_dict_parser_config=GlobalConfigDictParserConfig( + initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + ) + BaseNeMoGymCLIConfig.model_validate(global_config_dict) + + assert BENCHMARKS_DIR.exists(), "Missing benchmarks directory" + + config_paths = glob("**/config.yaml", root_dir=BENCHMARKS_DIR, recursive=True) + config_paths = [BENCHMARKS_DIR / p for p in config_paths] + config_paths = sorted(config_paths) + + benchmarks = _load_benchmarks_from_config_paths(config_paths) + + if not benchmarks: + rich.print("[yellow]No benchmarks found.[/yellow]") + rich.print(f"Expected benchmarks directory: {BENCHMARKS_DIR}") + return + + table = Table(title=f"Available benchmarks in NeMo Gym ({len(benchmarks)})") + table.add_column("Benchmark name") + table.add_column("Agent name") + table.add_column("Num repeats") + + for name, bench in benchmarks.items(): + table.add_row(name, bench.agent_name, str(bench.num_repeats)) + + rich.print(table) + + +class PrepareBenchmarkConfig(BaseNeMoGymCLIConfig): + """ + Prepare benchmark data by running the benchmark's prepare.py script. + + The benchmark is identified from a config_paths entry pointing to a + benchmarks/*/config.yaml file. + + Examples: + + ```bash + ng_prepare_benchmark "+config_paths=[benchmarks/aime24/config.yaml]" + ``` + """ + + use_cached_prepared_benchmarks: bool = Field( + default=False, description="Skip benchmark preparation if the prepared file is already present" + ) + num_prepare_benchmark_processes: int = Field( + default=1, description="Number of processes to parallelize benchmark preparation" + ) + + +def _multiprocess_benchmark_prepare_fn(args): + benchmark_config: BenchmarkConfig + prepare_module_path: str + (benchmark_config, prepare_module_path) = args + + print(f"Preparing benchmark: {benchmark_config.name}") + + module = importlib.import_module(prepare_module_path) + output_fpath = module.prepare() + assert output_fpath.absolute() == benchmark_config.dataset.jsonl_fpath.absolute(), ( + f"Expected the actual prepared dataset output fpath to match the jsonl_fpath set in the config. Instead got {output_fpath=} jsonl_fpath={benchmark_config.dataset.jsonl_fpath}" + ) + print(f"Benchmark data prepared at: {output_fpath}") + + +def prepare_benchmark() -> None: + """CLI command: prepare benchmark data.""" + global_config_dict = get_global_config_dict( + global_config_dict_parser_config=GlobalConfigDictParserConfig( + initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + ) + prepare_benchmark_config = PrepareBenchmarkConfig.model_validate(global_config_dict) + + benchmarks_dict: Dict[str, BenchmarkConfig] = dict() + for server_instance_name in global_config_dict: + server_config = global_config_dict[server_instance_name] + if not isinstance(server_config, (dict, DictConfig)) or "responses_api_agents" not in server_config: + continue + + inner_server_config = get_first_server_config_dict(global_config_dict, server_instance_name) + + datasets: List[BenchmarkDatasetConfig] = [] + for dataset in inner_server_config.get("datasets") or []: + if dataset["type"] != "benchmark": + continue + + datasets.append(BenchmarkDatasetConfig.model_validate(dataset)) + + if len(datasets) < 1: + continue + + assert len(datasets) == 1, ( + f"Expected 1 benchmark dataset for `{server_instance_name}`, but found {len(datasets)}!" + ) + + dataset = datasets[0] + + benchmarks_dict[server_instance_name] = BenchmarkConfig( + name=dataset.name, + path=Path(""), + agent_name=server_instance_name, + num_repeats=dataset.num_repeats, + dataset=dataset, + ) + + assert benchmarks_dict, ( + 'No benchmark config found in config_paths. Pass a benchmark config, e.g.: "+config_paths=[benchmarks/aime24/config.yaml]"' + ) + + # Validate all benchmarks before preparing any + prepare_script_missing: List[BenchmarkConfig] = [] + prepare_function_missing: List[BenchmarkConfig] = [] + + validated: List[Tuple[BenchmarkConfig, str]] = [] + already_prepared: List[BenchmarkConfig] = [] + for benchmark_config in benchmarks_dict.values(): + prepare_script_path = benchmark_config.dataset.prepare_script + if not prepare_script_path.exists(): + prepare_script_missing.append(benchmark_config) + continue + + prepare_module_path = ".".join(prepare_script_path.with_suffix("").parts) + module = importlib.import_module(prepare_module_path) + if not hasattr(module, "prepare"): + prepare_function_missing.append(benchmark_config) + continue + + is_already_prepared = benchmark_config.dataset.jsonl_fpath.exists() + if prepare_benchmark_config.use_cached_prepared_benchmarks and is_already_prepared: + already_prepared.append(benchmark_config) + continue + + validated.append((benchmark_config, prepare_module_path)) + + if already_prepared: + already_prepared_str = "".join(f"- {bc.name}: {bc.dataset.jsonl_fpath}\n" for bc in already_prepared) + already_prepared_str = f"""The following benchmarks have already been prepared. Since `use_cached_prepared_benchmarks=true`, we will skip re-preparation of those benchmarks. + {already_prepared_str}""" + print(already_prepared_str) + + errors_to_print = "" + if prepare_script_missing: + prepare_script_missing_str = "".join( + f"- {bc.name}: {bc.dataset.prepare_script}\n" for bc in prepare_script_missing + ) + errors_to_print += f"""The following benchmarks are missing a valid prepare script: +{prepare_script_missing_str} +""" + if prepare_function_missing: # pragma: no cover + prepare_function_missing_str = "".join( + f"- {bc.name}: {bc.dataset.prepare_script}\n" for bc in prepare_function_missing + ) + errors_to_print += f"""The following benchmarks have a prepare script, but are missing the prepare function: +{prepare_function_missing_str} +""" + if errors_to_print: + errors_to_print = f"""Did not prepare any benchmarks due to benchmark config errors. +{errors_to_print}""" + raise RuntimeError(errors_to_print) + + # Prepare after all validations pass + if prepare_benchmark_config.num_prepare_benchmark_processes > 1: # pragma: no cover + with Pool(processes=prepare_benchmark_config.num_prepare_benchmark_processes) as pool: + results = pool.imap_unordered(_multiprocess_benchmark_prepare_fn, validated) + list(tqdm(results, total=len(validated))) + else: + results = map(_multiprocess_benchmark_prepare_fn, validated) + list(tqdm(results, total=len(validated))) + + +def e2e_rollout_collection(): # pragma: no cover + global_config_dict = get_global_config_dict() + + # Ensure we have the right config first thing + e2e_rollout_collection_config = E2ERolloutCollectionConfig.model_validate(global_config_dict) + + # Prepare data + data_processor_config_dict = deepcopy(global_config_dict) + with open_dict(data_processor_config_dict): + data_processor_config_dict["should_download"] = True + data_processor_config_dict["mode"] = "train_preparation" + + output_fpath = Path(e2e_rollout_collection_config.output_jsonl_fpath) + data_process_output_dir = output_fpath.parent / "preprocessed_datasets" + data_processor_config_dict["output_dirpath"] = str(data_process_output_dir) + + input_jsonl_fpath = data_process_output_dir / f"{e2e_rollout_collection_config.split}.jsonl" + should_skip_data_processing = ( + e2e_rollout_collection_config.reuse_existing_data_preparation and input_jsonl_fpath.exists() + ) + if not should_skip_data_processing: + if e2e_rollout_collection_config.reuse_existing_data_preparation: + print( + f"Even though the `reuse_existing_data_preparation=true` flag was set, we will still do data preparation since the final input jsonl fpath `{input_jsonl_fpath}` does not exist yet" + ) + + data_processor = TrainDataProcessor() + data_processor.run(data_processor_config_dict) + else: + print( + f"Skipping data preparation since `reuse_existing_data_preparation=true` and the final input jsonl fpath `{input_jsonl_fpath}` already exists" + ) + + # Convert to RolloutCollectionConfig + rollout_collection_config_dict = deepcopy(global_config_dict) + with open_dict(rollout_collection_config_dict): + assert input_jsonl_fpath.exists(), input_jsonl_fpath + rollout_collection_config_dict["input_jsonl_fpath"] = str(input_jsonl_fpath) + + rollout_collection_config = RolloutCollectionConfig.model_validate( + OmegaConf.to_container(rollout_collection_config_dict) + ) + + rh = RunHelper() + rh.start(None) + + rch = RolloutCollectionHelper() + + print( + f"""Output artifacts: +1. Preprocessed datasets: {data_processor_config_dict["output_dirpath"]} +2. Dataset file used for rollout collection: {rollout_collection_config_dict["input_jsonl_fpath"]} +3. Rollout collection results file: {output_fpath} +""" + ) + try: + asyncio.run(rch.run_from_config(rollout_collection_config)) + except KeyboardInterrupt: + pass + finally: + rh.shutdown() + + +def collect_rollouts(): # pragma: no cover + config = RolloutCollectionConfig.model_validate(get_global_config_dict()) + rch = RolloutCollectionHelper() + + asyncio.run(rch.run_from_config(config)) + + +def aggregate_rollouts(): # pragma: no cover + config = RolloutAggregationConfig.model_validate(get_global_config_dict()) + rah = RolloutAggregationHelper() + + asyncio.run(rah.run_from_config(config)) + + +def reward_profile(): # pragma: no cover + config = RewardProfileConfig.model_validate(get_global_config_dict()) + + with open(config.materialized_inputs_jsonl_fpath) as f: + rows = list(map(orjson.loads, f)) + + with open(config.rollouts_jsonl_fpath) as f: + results = list(map(orjson.loads, f)) + + # Results may be out of order. + results.sort(key=lambda r: (r[TASK_INDEX_KEY_NAME], r[ROLLOUT_INDEX_KEY_NAME])) + + rp = RewardProfiler() + group_level_metrics, agent_level_metrics = rp.profile_from_data( + rows, results, allow_partial_rollouts=config.allow_partial_rollouts + ) + completion_summary = rp.profile_completion_summary(rows, results) + reward_profiling_fpath, agent_level_metrics_fpath = rp.write_to_disk( + group_level_metrics, agent_level_metrics, Path(config.rollouts_jsonl_fpath) + ) + + print(f"""Profiling outputs: +Reward profile completion: {completion_summary["completed_rollout_rows"]}/{completion_summary["expected_rollout_rows"]} rollout rows ({completion_summary["reward_profile_completion_pct"]:.2f}%) +Input rows: {completion_summary["total_input_rows"]} total; {completion_summary["complete_input_rows"]} complete; {completion_summary["partial_input_rows"]} partial; {completion_summary["missing_input_rows"]} without rollouts dropped from output. +Reward profiling outputs: {reward_profiling_fpath} +Agent-level metrics: {agent_level_metrics_fpath}""") diff --git a/nemo_gym/cli/general.py b/nemo_gym/cli/general.py new file mode 100644 index 0000000000..b085e97b3f --- /dev/null +++ b/nemo_gym/cli/general.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import platform +import sys +from importlib.metadata import entry_points +from importlib.metadata import version as md_version +from subprocess import Popen + +import psutil +import rich +from pydantic import Field + +from nemo_gym import PARENT_DIR, __version__ +from nemo_gym.config_types import BaseNeMoGymCLIConfig +from nemo_gym.global_config import get_global_config_dict + + +def display_help_legacy(): + """ + Display a list of available NeMo Gym CLI commands. + + Examples: + + ```bash + ng_help + ``` + """ + global_config_dict = get_global_config_dict() + # Just here for help + BaseNeMoGymCLIConfig.model_validate(global_config_dict) + + eps = entry_points().select(group="console_scripts") + project_scripts = {ep.name: ep.value for ep in eps if ep.name.startswith(("nemo_gym_", "ng_"))} + rich.print("""Run a command with `+h=true` or `+help=true` to see more detailed information! + +[bold]Available CLI scripts[/bold] +-----------------""") + for script in project_scripts: + if not script.startswith("ng_"): + continue + + print(script) + + +class VersionConfig(BaseNeMoGymCLIConfig): + """ + Display gym version and system information. + + Examples: + + ```bash + # Display version information + ng_version + + # Output as JSON + ng_version +json=true + ``` + """ + + json_format: bool = Field(default=False, alias="json", description="Output in JSON format for programmatic use.") + + +def version(): # pragma: no cover + """Display gym version and system information.""" + global_config_dict = get_global_config_dict() + config = VersionConfig.model_validate(global_config_dict) + + json_output = config.json_format + + version_info = { + "nemo_gym": __version__, + "python": platform.python_version(), + "python_path": sys.executable, + "installation_path": str(PARENT_DIR), + } + + key_deps = [ + "openai", + "ray", + ] + + dependencies = {dep: md_version(dep) for dep in key_deps} + + version_info["dependencies"] = dependencies + + # System info + version_info["system"] = { + "os": f"{platform.system()} {platform.release()}", + "platform": platform.platform(), + "architecture": platform.machine(), + "processor": platform.processor() or "unknown", + "cpus": os.cpu_count(), + } + + # Memory info + mem = psutil.virtual_memory() + version_info["system"]["memory_gb"] = round(mem.total / (1024**3), 2) + + # Output + if json_output: + print(json.dumps(version_info)) + else: + output = f"""\ +NeMo Gym v{version_info["nemo_gym"]} +Python {version_info["python"]} ({version_info["python_path"]}) +Installation: {version_info["installation_path"]}""" + + if "dependencies" in version_info: + deps_lines = "\n".join(f" {dep}: {ver}" for dep, ver in version_info["dependencies"].items()) + sys_info = version_info["system"] + output += f""" + +Key Dependencies: +{deps_lines} + +System: + OS: {sys_info["os"]} + Platform: {sys_info["platform"]} + Architecture: {sys_info["architecture"]} + Processor: {sys_info["processor"]} + CPUs: {sys_info["cpus"]} + Memory: {sys_info["memory_gb"]} GB""" + + print(output) + + +def reinstall(): # pragma: no cover + global_config_dict = get_global_config_dict() + # Just here for help + BaseNeMoGymCLIConfig.model_validate(global_config_dict) + + Popen("uv sync --extra dev --group docs", shell=True).communicate() diff --git a/nemo_gym/dataset_orchestrator.py b/nemo_gym/dataset_orchestrator.py index 80401564d7..a2cf9bdb14 100644 --- a/nemo_gym/dataset_orchestrator.py +++ b/nemo_gym/dataset_orchestrator.py @@ -15,15 +15,11 @@ from typing import Union from nemo_gym.config_types import ( - DeleteJsonlDatasetGitlabConfig, - DownloadJsonlDatasetHuggingFaceConfig, UploadJsonlDatasetHuggingFaceConfig, UploadJsonlDatasetHuggingFaceMaybeDeleteConfig, ) from nemo_gym.gitlab_utils import delete_model_from_gitlab, is_model_in_gitlab -from nemo_gym.hf_utils import download_hf_dataset_as_jsonl from nemo_gym.hf_utils import upload_jsonl_dataset as upload_jsonl_dataset_to_hf -from nemo_gym.server_utils import get_global_config_dict def delete_jsonl_dataset_from_gitlab(gitlab_model_name: str) -> None: # pragma: no cover @@ -57,33 +53,3 @@ def upload_jsonl_dataset_to_hf_maybe_delete( if delete_from_gitlab: delete_jsonl_dataset_from_gitlab(gitlab_model_name) - - -def upload_jsonl_dataset_to_hf_cli() -> None: # pragma: no cover - global_config = get_global_config_dict() - config = UploadJsonlDatasetHuggingFaceMaybeDeleteConfig.model_validate(global_config) - upload_jsonl_dataset_to_hf_maybe_delete(config, delete_from_gitlab=config.delete_from_gitlab) - - -def upload_jsonl_dataset_to_hf_and_delete_gitlab_cli() -> None: # pragma: no cover - global_config = get_global_config_dict() - config = UploadJsonlDatasetHuggingFaceConfig.model_validate(global_config) - upload_jsonl_dataset_to_hf_maybe_delete(config, delete_from_gitlab=True) - - -def download_jsonl_dataset_from_hf_cli() -> None: # pragma: no cover - global_config = get_global_config_dict() - config = DownloadJsonlDatasetHuggingFaceConfig.model_validate(global_config) - - if config.artifact_fpath: - print(f"Downloading file '{config.artifact_fpath}' from '{config.repo_id}'...") - else: - print(f"Downloading '{config.split or 'all'}' split(s) from '{config.repo_id}'...") - - download_hf_dataset_as_jsonl(config) - - -def delete_jsonl_dataset_from_gitlab_cli() -> None: # pragma: no cover - global_config = get_global_config_dict() - config = DeleteJsonlDatasetGitlabConfig.model_validate(global_config) - delete_jsonl_dataset_from_gitlab(config.dataset_name) diff --git a/nemo_gym/gitlab_utils.py b/nemo_gym/gitlab_utils.py index 23c7a2898f..e83b85787d 100644 --- a/nemo_gym/gitlab_utils.py +++ b/nemo_gym/gitlab_utils.py @@ -74,12 +74,6 @@ def upload_jsonl_dataset( """) -def upload_jsonl_dataset_cli() -> None: # pragma: no cover - global_config = get_global_config_dict() - config = UploadJsonlDatasetGitlabConfig.model_validate(global_config) - upload_jsonl_dataset(config) - - def download_jsonl_dataset( config: DownloadJsonlDatasetGitlabConfig, ) -> None: # pragma: no cover @@ -100,12 +94,6 @@ def download_jsonl_dataset( f.write(response.content.decode()) -def download_jsonl_dataset_cli() -> None: # pragma: no cover - global_config = get_global_config_dict() - config = DownloadJsonlDatasetGitlabConfig.model_validate(global_config) - download_jsonl_dataset(config) - - def is_model_in_gitlab(model_name: str) -> bool: # pragma: no cover client = create_mlflow_client() diff --git a/nemo_gym/prompt.py b/nemo_gym/prompt.py index 8a4d10f5bc..fae834d7ca 100644 --- a/nemo_gym/prompt.py +++ b/nemo_gym/prompt.py @@ -29,7 +29,6 @@ from nemo_gym import PARENT_DIR from nemo_gym.config_types import BaseNeMoGymCLIConfig -from nemo_gym.global_config import GlobalConfigDictParserConfig, get_global_config_dict class PromptConfig(BaseModel): @@ -162,14 +161,3 @@ class MaterializePromptsConfig(BaseNeMoGymCLIConfig): input_jsonl_fpath: str = Field(description="Raw JSONL data (no responses_create_params.input).") prompt_config: str = Field(description="Path to prompt YAML file to apply.") output_jsonl_fpath: str = Field(description="Output path for materialized JSONL with populated prompts.") - - -def materialize_prompts_cli() -> None: # pragma: no cover - """CLI entry point for ng_materialize_prompts.""" - global_config_dict = get_global_config_dict( - global_config_dict_parser_config=GlobalConfigDictParserConfig( - initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, - ) - ) - config = MaterializePromptsConfig.model_validate(global_config_dict) - materialize_prompts(config.input_jsonl_fpath, config.prompt_config, config.output_jsonl_fpath) diff --git a/nemo_gym/reward_profile.py b/nemo_gym/reward_profile.py index e7bc8eb6d6..de395f4798 100644 --- a/nemo_gym/reward_profile.py +++ b/nemo_gym/reward_profile.py @@ -31,7 +31,6 @@ AGENT_REF_KEY_NAME, ROLLOUT_INDEX_KEY_NAME, TASK_INDEX_KEY_NAME, - get_global_config_dict, ) @@ -701,31 +700,3 @@ def compute_aggregate_metrics( agent_metrics=serialized_agent, key_metrics=key_metrics, ) - - -def reward_profile(): # pragma: no cover - config = RewardProfileConfig.model_validate(get_global_config_dict()) - - with open(config.materialized_inputs_jsonl_fpath) as f: - rows = list(map(orjson.loads, f)) - - with open(config.rollouts_jsonl_fpath) as f: - results = list(map(orjson.loads, f)) - - # Results may be out of order. - results.sort(key=lambda r: (r[TASK_INDEX_KEY_NAME], r[ROLLOUT_INDEX_KEY_NAME])) - - rp = RewardProfiler() - group_level_metrics, agent_level_metrics = rp.profile_from_data( - rows, results, allow_partial_rollouts=config.allow_partial_rollouts - ) - completion_summary = rp.profile_completion_summary(rows, results) - reward_profiling_fpath, agent_level_metrics_fpath = rp.write_to_disk( - group_level_metrics, agent_level_metrics, Path(config.rollouts_jsonl_fpath) - ) - - print(f"""Profiling outputs: -Reward profile completion: {completion_summary["completed_rollout_rows"]}/{completion_summary["expected_rollout_rows"]} rollout rows ({completion_summary["reward_profile_completion_pct"]:.2f}%) -Input rows: {completion_summary["total_input_rows"]} total; {completion_summary["complete_input_rows"]} complete; {completion_summary["partial_input_rows"]} partial; {completion_summary["missing_input_rows"]} without rollouts dropped from output. -Reward profiling outputs: {reward_profiling_fpath} -Agent-level metrics: {agent_level_metrics_fpath}""") diff --git a/nemo_gym/rollout_collection.py b/nemo_gym/rollout_collection.py index 86ae07d1ff..ed53bf4904 100644 --- a/nemo_gym/rollout_collection.py +++ b/nemo_gym/rollout_collection.py @@ -44,7 +44,6 @@ from nemo_gym.server_utils import ( GlobalAIOHTTPAsyncClientConfig, ServerClient, - get_global_config_dict, get_response_json, is_global_aiohttp_client_request_debug_enabled, is_global_aiohttp_client_setup, @@ -617,13 +616,6 @@ def setup_server_client( return server_client -def collect_rollouts(): # pragma: no cover - config = RolloutCollectionConfig.model_validate(get_global_config_dict()) - rch = RolloutCollectionHelper() - - asyncio.run(rch.run_from_config(config)) - - class RolloutAggregationConfig(BaseNeMoGymCLIConfig): """ Aggregate metrics across rollout shards produced by `ng_collect_rollouts +disable_aggregation=true`. @@ -719,10 +711,3 @@ async def run_from_config(self, config: RolloutAggregationConfig) -> Optional[Pa Aggregate metrics: {aggregate_metrics_fpath}""") return aggregate_metrics_fpath - - -def aggregate_rollouts(): # pragma: no cover - config = RolloutAggregationConfig.model_validate(get_global_config_dict()) - rah = RolloutAggregationHelper() - - asyncio.run(rah.run_from_config(config)) diff --git a/nemo_gym/train_data_utils.py b/nemo_gym/train_data_utils.py index 5191741b37..2ea6e4842d 100644 --- a/nemo_gym/train_data_utils.py +++ b/nemo_gym/train_data_utils.py @@ -40,7 +40,6 @@ from nemo_gym.global_config import ( HF_TOKEN_KEY_NAME, GlobalConfigDictParser, - GlobalConfigDictParserConfig, get_global_config_dict, ) from nemo_gym.hf_utils import ( @@ -823,14 +822,3 @@ def validate_backend_credentials(backend: str) -> tuple[bool, str]: ) return True, "" - - -def prepare_data(): # pragma: no cover - global_config_dict = get_global_config_dict( - global_config_dict_parser_config=GlobalConfigDictParserConfig( - initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, - ) - ) - - data_processor = TrainDataProcessor() - data_processor.run(global_config_dict) diff --git a/pyproject.toml b/pyproject.toml index ac4f4ffacc..4a52c6932c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -312,86 +312,86 @@ Homepage = "https://github.com/NVIDIA-NeMo/Gym" [project.scripts] # Run/test scripts for servers. -nemo_gym_run = "nemo_gym.cli:run" -ng_run = "nemo_gym.cli:run" -nemo_gym_test = "nemo_gym.cli:test" -ng_test = "nemo_gym.cli:test" -nemo_gym_test_all = "nemo_gym.cli:test_all" -ng_test_all = "nemo_gym.cli:test_all" +nemo_gym_run = "nemo_gym.cli.env:run" +ng_run = "nemo_gym.cli.env:run" +nemo_gym_test = "nemo_gym.cli.env:test" +ng_test = "nemo_gym.cli.env:test" +nemo_gym_test_all = "nemo_gym.cli.env:test_all" +ng_test_all = "nemo_gym.cli.env:test_all" # Development convenience aliases. -nemo_gym_dev_test = "nemo_gym.cli:dev_test" -ng_dev_test = "nemo_gym.cli:dev_test" -nemo_gym_init_resources_server = "nemo_gym.cli:init_resources_server" -ng_init_resources_server = "nemo_gym.cli:init_resources_server" +nemo_gym_dev_test = "nemo_gym.cli.dev:dev_test" +ng_dev_test = "nemo_gym.cli.dev:dev_test" +nemo_gym_init_resources_server = "nemo_gym.cli.env:init_resources_server" +ng_init_resources_server = "nemo_gym.cli.env:init_resources_server" # Benchmarks -nemo_gym_list_benchmarks = "nemo_gym.benchmarks:list_benchmarks" -ng_list_benchmarks = "nemo_gym.benchmarks:list_benchmarks" -nemo_gym_prepare_benchmark = "nemo_gym.benchmarks:prepare_benchmark" -ng_prepare_benchmark = "nemo_gym.benchmarks:prepare_benchmark" +nemo_gym_list_benchmarks = "nemo_gym.cli.eval:list_benchmarks" +ng_list_benchmarks = "nemo_gym.cli.eval:list_benchmarks" +nemo_gym_prepare_benchmark = "nemo_gym.cli.eval:prepare_benchmark" +ng_prepare_benchmark = "nemo_gym.cli.eval:prepare_benchmark" # Rollout collection -nemo_gym_collect_rollouts = "nemo_gym.rollout_collection:collect_rollouts" -ng_collect_rollouts = "nemo_gym.rollout_collection:collect_rollouts" -nemo_gym_e2e_collect_rollouts = "nemo_gym.cli:e2e_rollout_collection" -ng_e2e_collect_rollouts = "nemo_gym.cli:e2e_rollout_collection" -nemo_gym_aggregate_rollouts = "nemo_gym.rollout_collection:aggregate_rollouts" -ng_aggregate_rollouts = "nemo_gym.rollout_collection:aggregate_rollouts" +nemo_gym_collect_rollouts = "nemo_gym.cli.eval:collect_rollouts" +ng_collect_rollouts = "nemo_gym.cli.eval:collect_rollouts" +nemo_gym_e2e_collect_rollouts = "nemo_gym.cli.eval:e2e_rollout_collection" +ng_e2e_collect_rollouts = "nemo_gym.cli.eval:e2e_rollout_collection" +nemo_gym_aggregate_rollouts = "nemo_gym.cli.eval:aggregate_rollouts" +ng_aggregate_rollouts = "nemo_gym.cli.eval:aggregate_rollouts" # Prompt materialization -nemo_gym_materialize_prompts = "nemo_gym.prompt:materialize_prompts_cli" -ng_materialize_prompts = "nemo_gym.prompt:materialize_prompts_cli" +nemo_gym_materialize_prompts = "nemo_gym.cli.dataset:materialize_prompts_cli" +ng_materialize_prompts = "nemo_gym.cli.dataset:materialize_prompts_cli" # Reward profiling -nemo_gym_reward_profile = "nemo_gym.reward_profile:reward_profile" -ng_reward_profile = "nemo_gym.reward_profile:reward_profile" +nemo_gym_reward_profile = "nemo_gym.cli.eval:reward_profile" +ng_reward_profile = "nemo_gym.cli.eval:reward_profile" # Dataset management -nemo_gym_upload_dataset_to_gitlab = "nemo_gym.gitlab_utils:upload_jsonl_dataset_cli" -ng_upload_dataset_to_gitlab = "nemo_gym.gitlab_utils:upload_jsonl_dataset_cli" -nemo_gym_download_dataset_from_gitlab = "nemo_gym.gitlab_utils:download_jsonl_dataset_cli" -ng_download_dataset_from_gitlab = "nemo_gym.gitlab_utils:download_jsonl_dataset_cli" -nemo_gym_prepare_data = "nemo_gym.train_data_utils:prepare_data" -ng_prepare_data = "nemo_gym.train_data_utils:prepare_data" +nemo_gym_upload_dataset_to_gitlab = "nemo_gym.cli.dataset:upload_jsonl_dataset_cli" +ng_upload_dataset_to_gitlab = "nemo_gym.cli.dataset:upload_jsonl_dataset_cli" +nemo_gym_download_dataset_from_gitlab = "nemo_gym.cli.dataset:download_jsonl_dataset_cli" +ng_download_dataset_from_gitlab = "nemo_gym.cli.dataset:download_jsonl_dataset_cli" +nemo_gym_prepare_data = "nemo_gym.cli.dataset:prepare_data" +ng_prepare_data = "nemo_gym.cli.dataset:prepare_data" # HF dataset upload and download -nemo_gym_upload_dataset_to_hf = "nemo_gym.dataset_orchestrator:upload_jsonl_dataset_to_hf_cli" -ng_upload_dataset_to_hf = "nemo_gym.dataset_orchestrator:upload_jsonl_dataset_to_hf_cli" -nemo_gym_download_dataset_from_hf = "nemo_gym.dataset_orchestrator:download_jsonl_dataset_from_hf_cli" -ng_download_dataset_from_hf = "nemo_gym.dataset_orchestrator:download_jsonl_dataset_from_hf_cli" +nemo_gym_upload_dataset_to_hf = "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli" +ng_upload_dataset_to_hf = "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli" +nemo_gym_download_dataset_from_hf = "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli" +ng_download_dataset_from_hf = "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli" # Gitlab -> HF (upload then delete) -nemo_gym_gitlab_to_hf_dataset = "nemo_gym.dataset_orchestrator:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli" -ng_gitlab_to_hf_dataset = "nemo_gym.dataset_orchestrator:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli" +nemo_gym_gitlab_to_hf_dataset = "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli" +ng_gitlab_to_hf_dataset = "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli" # Manually delete from Gitlab -nemo_gym_delete_dataset_from_gitlab = "nemo_gym.dataset_orchestrator:delete_jsonl_dataset_from_gitlab_cli" -ng_delete_dataset_from_gitlab = "nemo_gym.dataset_orchestrator:delete_jsonl_dataset_from_gitlab_cli" +nemo_gym_delete_dataset_from_gitlab = "nemo_gym.cli.dataset:delete_jsonl_dataset_from_gitlab_cli" +ng_delete_dataset_from_gitlab = "nemo_gym.cli.dataset:delete_jsonl_dataset_from_gitlab_cli" # Configuration utils -nemo_gym_dump_config = "nemo_gym.cli:dump_config" -ng_dump_config = "nemo_gym.cli:dump_config" +nemo_gym_dump_config = "nemo_gym.cli.env:dump_config" +ng_dump_config = "nemo_gym.cli.env:dump_config" # Display help -nemo_gym_help = "nemo_gym.cli:display_help" -ng_help = "nemo_gym.cli:display_help" +nemo_gym_help = "nemo_gym.cli.general:display_help_legacy" +ng_help = "nemo_gym.cli.general:display_help_legacy" # Server status -nemo_gym_status = "nemo_gym.cli:status" -ng_status = "nemo_gym.cli:status" +nemo_gym_status = "nemo_gym.cli.env:status" +ng_status = "nemo_gym.cli.env:status" # Environment-specific uv pip list -nemo_gym_pip_list = "nemo_gym.cli:pip_list" -ng_pip_list = "nemo_gym.cli:pip_list" +nemo_gym_pip_list = "nemo_gym.cli.env:pip_list" +ng_pip_list = "nemo_gym.cli.env:pip_list" # Display version -nemo_gym_version = "nemo_gym.cli:version" -ng_version = "nemo_gym.cli:version" +nemo_gym_version = "nemo_gym.cli.general:version" +ng_version = "nemo_gym.cli.general:version" # Re-install Gym and dependencies -nemo_gym_reinstall = "nemo_gym.cli:reinstall" -ng_reinstall = "nemo_gym.cli:reinstall" +nemo_gym_reinstall = "nemo_gym.cli.general:reinstall" +ng_reinstall = "nemo_gym.cli.general:reinstall" [tool.setuptools.packages.find] where = ["."] diff --git a/tests/unit_tests/test_benchmarks.py b/tests/unit_tests/test_benchmarks.py index 02e8ecb9cb..653788858f 100644 --- a/tests/unit_tests/test_benchmarks.py +++ b/tests/unit_tests/test_benchmarks.py @@ -19,7 +19,7 @@ from omegaconf import OmegaConf from yaml import safe_load -from nemo_gym.benchmarks import list_benchmarks, prepare_benchmark +from nemo_gym.cli.eval import list_benchmarks, prepare_benchmark def _mock_global_config(config: dict = None): @@ -29,14 +29,14 @@ def _mock_global_config(config: dict = None): class TestListBenchmarks: def test_lists_found_benchmarks(self, capsys) -> None: - with patch("nemo_gym.benchmarks.get_global_config_dict", return_value=_mock_global_config()): + with patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config()): list_benchmarks() assert "aime24" in capsys.readouterr().out def test_no_benchmarks(self, capsys) -> None: with ( - patch("nemo_gym.benchmarks.get_global_config_dict", return_value=_mock_global_config()), - patch("nemo_gym.benchmarks._load_benchmarks_from_config_paths", return_value={}), + patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config()), + patch("nemo_gym.cli.eval._load_benchmarks_from_config_paths", return_value={}), ): list_benchmarks() assert "No benchmarks found" in capsys.readouterr().out @@ -73,13 +73,13 @@ def test_calls_prepare(self, tmp_path: Path) -> None: with ( patch( - "nemo_gym.benchmarks.get_global_config_dict", + "nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config( {"config_paths": [str(config_path)], **safe_load(config_path.read_text())} ), ), - patch("nemo_gym.benchmarks.BENCHMARKS_DIR", bench_dir.parent), - patch("nemo_gym.benchmarks.importlib.import_module", return_value=mock_module), + patch("nemo_gym.cli.eval.BENCHMARKS_DIR", bench_dir.parent), + patch("nemo_gym.cli.eval.importlib.import_module", return_value=mock_module), ): prepare_benchmark() mock_module.prepare.assert_called_once() @@ -90,12 +90,12 @@ def test_missing_prepare_py(self, tmp_path: Path) -> None: with ( patch( - "nemo_gym.benchmarks.get_global_config_dict", + "nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config( {"config_paths": [str(config_path)], **safe_load(config_path.read_text())} ), ), - patch("nemo_gym.benchmarks.BENCHMARKS_DIR", bench_dir.parent), + patch("nemo_gym.cli.eval.BENCHMARKS_DIR", bench_dir.parent), ): with pytest.raises(RuntimeError, match="The following benchmarks are missing a valid prepare script"): prepare_benchmark() @@ -107,13 +107,13 @@ def test_missing_prepare_function(self, tmp_path: Path) -> None: with ( patch( - "nemo_gym.benchmarks.get_global_config_dict", + "nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config( {"config_paths": [str(config_path)], **safe_load(config_path.read_text())} ), ), - patch("nemo_gym.benchmarks.BENCHMARKS_DIR", bench_dir.parent), - patch("nemo_gym.benchmarks.importlib.import_module", return_value=mock_module), + patch("nemo_gym.cli.eval.BENCHMARKS_DIR", bench_dir.parent), + patch("nemo_gym.cli.eval.importlib.import_module", return_value=mock_module), ): with pytest.raises( AssertionError, @@ -124,10 +124,10 @@ def test_missing_prepare_function(self, tmp_path: Path) -> None: def test_no_benchmark_in_config_paths(self) -> None: with ( patch( - "nemo_gym.benchmarks.get_global_config_dict", + "nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config({"config_paths": ["resources_servers/foo/configs/foo.yaml"]}), ), - patch("nemo_gym.benchmarks._load_benchmarks_from_config_paths", return_value={}), + patch("nemo_gym.cli.eval._load_benchmarks_from_config_paths", return_value={}), ): with pytest.raises(AssertionError, match="No benchmark config found in config_paths"): prepare_benchmark() @@ -141,7 +141,7 @@ def test_caching_sanity(self, tmp_path: Path) -> None: with ( patch( - "nemo_gym.benchmarks.get_global_config_dict", + "nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config( { "use_cached_prepared_benchmarks": True, @@ -150,8 +150,8 @@ def test_caching_sanity(self, tmp_path: Path) -> None: } ), ), - patch("nemo_gym.benchmarks.BENCHMARKS_DIR", bench_dir.parent), - patch("nemo_gym.benchmarks.importlib.import_module", return_value=mock_module), + patch("nemo_gym.cli.eval.BENCHMARKS_DIR", bench_dir.parent), + patch("nemo_gym.cli.eval.importlib.import_module", return_value=mock_module), ): prepare_benchmark() diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index 74f7fce578..91106ec850 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -26,14 +26,14 @@ import nemo_gym.global_config from nemo_gym import PARENT_DIR -from nemo_gym.cli import ( +from nemo_gym.cli.env import ( _FORCE_KILL_REAP_TIMEOUT_SEC, _GRACEFUL_SHUTDOWN_TIMEOUT_SEC, RunConfig, RunHelper, - display_help, init_resources_server, ) +from nemo_gym.cli.general import display_help_legacy from nemo_gym.config_types import ResourcesServerInstanceConfig @@ -76,7 +76,7 @@ def test_display_help_discovers_scripts(self) -> None: text_trap = StringIO() mp.setattr(sys, "stdout", text_trap) - display_help() + display_help_legacy() output = text_trap.getvalue() assert "ng_help" in output diff --git a/tests/unit_tests/test_server_status.py b/tests/unit_tests/test_server_status.py index 609ddbeae9..e7fdd26dd5 100644 --- a/tests/unit_tests/test_server_status.py +++ b/tests/unit_tests/test_server_status.py @@ -18,9 +18,8 @@ import requests from pytest import MonkeyPatch -from nemo_gym.cli import ServerInstanceDisplayConfig from nemo_gym.server_status import StatusCommand -from nemo_gym.server_utils import ServerClient +from nemo_gym.server_utils import ServerClient, ServerInstanceDisplayConfig class TestServerStatus: From 7cdcb8e2350c089b83d0d8c9828ba64c49d80911 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Fri, 12 Jun 2026 15:01:16 +0200 Subject: [PATCH 02/40] feat: add basic 'gym' command router Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 208 +++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 6 ++ 2 files changed, 214 insertions(+) create mode 100644 nemo_gym/cli/main.py diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py new file mode 100644 index 0000000000..76c2a73630 --- /dev/null +++ b/nemo_gym/cli/main.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import importlib +import sys +from collections.abc import Callable +from dataclasses import dataclass + + +VERSION_TARGET = "nemo_gym.cli.general:version" + + +@dataclass(frozen=True) +class Command: + # What to run: either a "module:function" string (lazily imported and called with no args), + # or a callable(args, overrides) that owns dispatch (e.g. picks the target from parsed flags). + target: str | Callable[[argparse.Namespace, list[str]], None] + # One-line help shown in the parent listing and atop this command's own --help. + summary: str | None = None + # Hook to add this command's own flags to its subparser. + configure: Callable[[argparse.ArgumentParser], None] | None = None + + +# Flags shared by all commands are added here once and attached via `parents=[COMMON]`. +COMMON = argparse.ArgumentParser(add_help=False) + + +def dispatch(target: str, overrides: list[str]) -> None: + module_path, func_name = target.split(":") + # Drop the parsed command tokens so the downstream Hydra parsing only sees overrides. + sys.argv = [sys.argv[0], *overrides] + func = getattr(importlib.import_module(module_path), func_name) + func() + + +def _toggle_command(flag: str, flag_help: str, *, off_command: str, on_command: str, summary: str) -> Command: + """Build a command whose target switches from `off` (default) to `on` when -- is given.""" + dest = flag.replace("-", "_") + + def configure(parser: argparse.ArgumentParser) -> None: + parser.add_argument(f"--{flag}", action="store_true", help=flag_help) + + def target(args: argparse.Namespace, overrides: list[str]) -> None: + dispatch(on_command if getattr(args, dest) else off_command, overrides) + + return Command(target=target, configure=configure, summary=summary) + + +def _choice_command(flag: str, flag_help: str, *, targets: dict[str, str], default: str, summary: str) -> Command: + """Build a command whose target is selected by -- {choices} (default `default`).""" + dest = flag.replace("-", "_") + + def configure(parser: argparse.ArgumentParser) -> None: + parser.add_argument(f"--{flag}", choices=tuple(targets), default=default, help=flag_help) + + def target(args: argparse.Namespace, overrides: list[str]) -> None: + dispatch(targets[getattr(args, dest)], overrides) + + return Command(target=target, configure=configure, summary=summary) + + +# One-line help for each command group, shown in `gym --help`. +GROUPS = { + "list": "List available components. As of now, only benchmarks are available.", + "dataset": "Manage datasets.", + "env": "Develop and run environments.", + "eval": "Run evaluations.", + "dev": "Contributor helpers.", +} + +COMMANDS = { + "list benchmarks": Command(target="nemo_gym.cli.eval:list_benchmarks", summary="List available benchmarks."), + "dataset upload": _choice_command( + "target", + "Destination backend (default: hf).", + targets={ + "hf": "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli", + "gitlab": "nemo_gym.cli.dataset:upload_jsonl_dataset_cli", + }, + default="hf", + summary="Upload a prepared dataset to HF (default) or GitLab.", + ), + "dataset download": _choice_command( + "source", + "Source backend (default: hf).", + targets={ + "hf": "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli", + "gitlab": "nemo_gym.cli.dataset:download_jsonl_dataset_cli", + }, + default="hf", + summary="Download a dataset from HF (default) or GitLab.", + ), + "dataset rm": Command( + target="nemo_gym.cli.dataset:delete_jsonl_dataset_from_gitlab_cli", + summary="Delete a dataset from GitLab.", + ), + "dataset migrate": Command( + target="nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli", + summary="Transfer a dataset from GitLab to HF.", + ), + "dataset render": Command( + target="nemo_gym.cli.dataset:materialize_prompts_cli", + summary="Generate a dataset preview.", + ), + "dataset collate": Command( + target="nemo_gym.cli.dataset:prepare_data", + summary="Validate and collate the dataset.", + ), + "env init": Command( + target="nemo_gym.cli.env:init_resources_server", + summary="Scaffold config for a new server, benchmark, or agent.", + ), + "env resolve": Command( + target="nemo_gym.cli.env:dump_config", + summary="Resolve the final config from configs, flags, and overrides.", + ), + "env packages": Command( + target="nemo_gym.cli.env:pip_list", + summary="Print pip packages for the selected resource server.", + ), + "env test": Command(target="nemo_gym.cli.env:test", summary="Test the resource server(s)."), + "env run": Command(target="nemo_gym.cli.env:run", summary="Start the servers."), + "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status."), + "eval prepare": Command( + target="nemo_gym.cli.eval:prepare_benchmark", + summary="Prepare benchmark data and dump it to disk.", + ), + "eval run": _toggle_command( + "no-serve", + "Collect against already-running servers instead of starting them.", + off_command="nemo_gym.cli.eval:e2e_rollout_collection", + on_command="nemo_gym.cli.eval:collect_rollouts", + summary="Collate data, start servers, and collect rollouts.", + ), + "eval aggregate": Command( + target="nemo_gym.cli.eval:aggregate_rollouts", + summary="Aggregate sharded rollout results.", + ), + "eval profile": Command( + target="nemo_gym.cli.eval:reward_profile", + summary="Compute a reward profile from rollouts.", + ), + "dev test": Command(target="nemo_gym.cli.dev:dev_test", summary="Run NeMo Gym's unit tests."), +} + + +def _add_leaf(subparsers: argparse._SubParsersAction, name: str, command: Command) -> None: + leaf = subparsers.add_parser(name, parents=[COMMON], help=command.summary, description=command.summary) + leaf.set_defaults(_command=command) + if command.configure is not None: + command.configure(leaf) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="gym", add_help=True) + parser.add_argument("--version", action="store_true", help="Show the NeMo Gym version and exit.") + parser.set_defaults(_parser=parser) + + subparsers = parser.add_subparsers() + groups: dict[str, argparse._SubParsersAction] = {} + + for command_name, command in COMMANDS.items(): + parts = command_name.split() + if len(parts) == 1: + _add_leaf(subparsers, parts[0], command) + continue + + group_name, action_name = parts + if group_name not in groups: + group_parser = subparsers.add_parser( + group_name, help=GROUPS.get(group_name), description=GROUPS.get(group_name) + ) + group_parser.set_defaults(_parser=group_parser) + groups[group_name] = group_parser.add_subparsers() + _add_leaf(groups[group_name], action_name, command) + + return parser + + +def main() -> None: + parser = build_parser() + args, overrides = parser.parse_known_args() + + if args.version: + dispatch(VERSION_TARGET, overrides) + return + + command = getattr(args, "_command", None) + if command is None: + args._parser.print_help() + sys.exit(1) + + if callable(command.target): + command.target(args, overrides) + else: + dispatch(command.target, overrides) diff --git a/pyproject.toml b/pyproject.toml index 4a52c6932c..5196613221 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -311,6 +311,12 @@ Download = "https://github.com/NVIDIA-NeMo/Gym/releases" Homepage = "https://github.com/NVIDIA-NeMo/Gym" [project.scripts] + +gym = "nemo_gym.cli.main:main" +ng = "nemo_gym.cli.main:main" + +########################## DEPRECATED COMMANDS ########################## + # Run/test scripts for servers. nemo_gym_run = "nemo_gym.cli.env:run" ng_run = "nemo_gym.cli.env:run" From cba678a198a263a80d613e5e29ec5d865580f82d Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Mon, 15 Jun 2026 13:23:16 +0200 Subject: [PATCH 03/40] feat: add --config and --storage flags to the gym router Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 115 +++++++++++++++++++++++++------------------ 1 file changed, 67 insertions(+), 48 deletions(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 76c2a73630..d8fd2ad925 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -16,12 +16,20 @@ import importlib import sys from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field VERSION_TARGET = "nemo_gym.cli.general:version" +@dataclass(frozen=True) +class Flag: + # Register this flag's argument(s) on a command's subparser. + register: Callable[[argparse.ArgumentParser], None] + # Turn the parsed value into leading Hydra override tokens (default: contributes nothing). + translate_to_hydra: Callable[[argparse.Namespace], list[str]] = lambda args: [] + + @dataclass(frozen=True) class Command: # What to run: either a "module:function" string (lazily imported and called with no args), @@ -29,12 +37,8 @@ class Command: target: str | Callable[[argparse.Namespace, list[str]], None] # One-line help shown in the parent listing and atop this command's own --help. summary: str | None = None - # Hook to add this command's own flags to its subparser. - configure: Callable[[argparse.ArgumentParser], None] | None = None - - -# Flags shared by all commands are added here once and attached via `parents=[COMMON]`. -COMMON = argparse.ArgumentParser(add_help=False) + # Flags this command accepts; reusable ones (e.g. CONFIG) are shared across commands. + flags: tuple[Flag, ...] = field(default_factory=tuple) def dispatch(target: str, overrides: list[str]) -> None: @@ -45,30 +49,52 @@ def dispatch(target: str, overrides: list[str]) -> None: func() -def _toggle_command(flag: str, flag_help: str, *, off_command: str, on_command: str, summary: str) -> Command: - """Build a command whose target switches from `off` (default) to `on` when -- is given.""" - dest = flag.replace("-", "_") +# Shared flag: load Gym config files. Reused by every command that reads server/benchmark configs. +CONFIG = Flag( + register=lambda p: p.add_argument( + "--config", + action="append", + metavar="PATH", + help="Config file to load; repeatable. Maps to +config_paths=[...].", + ), + translate_to_hydra=lambda args: [f"+config_paths=[{','.join(args.config)}]"] if args.config else [], +) - def configure(parser: argparse.ArgumentParser) -> None: - parser.add_argument(f"--{flag}", action="store_true", help=flag_help) +# Command-specific flags read by the corresponding target callable below. +NO_SERVE = Flag( + register=lambda p: p.add_argument( + "--no-serve", + action="store_true", + help="Collect against already-running servers instead of starting them.", + ) +) - def target(args: argparse.Namespace, overrides: list[str]) -> None: - dispatch(on_command if getattr(args, dest) else off_command, overrides) +STORAGE = Flag( + register=lambda p: p.add_argument( + "--storage", choices=("hf", "gitlab"), default="hf", help="Storage backend (default: hf)." + ) +) - return Command(target=target, configure=configure, summary=summary) +def _eval_run(args: argparse.Namespace, overrides: list[str]) -> None: + target = "nemo_gym.cli.eval:collect_rollouts" if args.no_serve else "nemo_gym.cli.eval:e2e_rollout_collection" + dispatch(target, overrides) -def _choice_command(flag: str, flag_help: str, *, targets: dict[str, str], default: str, summary: str) -> Command: - """Build a command whose target is selected by -- {choices} (default `default`).""" - dest = flag.replace("-", "_") - def configure(parser: argparse.ArgumentParser) -> None: - parser.add_argument(f"--{flag}", choices=tuple(targets), default=default, help=flag_help) +def _dataset_upload(args: argparse.Namespace, overrides: list[str]) -> None: + targets = { + "hf": "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli", + "gitlab": "nemo_gym.cli.dataset:upload_jsonl_dataset_cli", + } + dispatch(targets[args.storage], overrides) - def target(args: argparse.Namespace, overrides: list[str]) -> None: - dispatch(targets[getattr(args, dest)], overrides) - return Command(target=target, configure=configure, summary=summary) +def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: + targets = { + "hf": "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli", + "gitlab": "nemo_gym.cli.dataset:download_jsonl_dataset_cli", + } + dispatch(targets[args.storage], overrides) # One-line help for each command group, shown in `gym --help`. @@ -82,25 +108,15 @@ def target(args: argparse.Namespace, overrides: list[str]) -> None: COMMANDS = { "list benchmarks": Command(target="nemo_gym.cli.eval:list_benchmarks", summary="List available benchmarks."), - "dataset upload": _choice_command( - "target", - "Destination backend (default: hf).", - targets={ - "hf": "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli", - "gitlab": "nemo_gym.cli.dataset:upload_jsonl_dataset_cli", - }, - default="hf", + "dataset upload": Command( + target=_dataset_upload, summary="Upload a prepared dataset to HF (default) or GitLab.", + flags=(STORAGE,), ), - "dataset download": _choice_command( - "source", - "Source backend (default: hf).", - targets={ - "hf": "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli", - "gitlab": "nemo_gym.cli.dataset:download_jsonl_dataset_cli", - }, - default="hf", + "dataset download": Command( + target=_dataset_download, summary="Download a dataset from HF (default) or GitLab.", + flags=(STORAGE,), ), "dataset rm": Command( target="nemo_gym.cli.dataset:delete_jsonl_dataset_from_gitlab_cli", @@ -117,6 +133,7 @@ def target(args: argparse.Namespace, overrides: list[str]) -> None: "dataset collate": Command( target="nemo_gym.cli.dataset:prepare_data", summary="Validate and collate the dataset.", + flags=(CONFIG,), ), "env init": Command( target="nemo_gym.cli.env:init_resources_server", @@ -125,28 +142,29 @@ def target(args: argparse.Namespace, overrides: list[str]) -> None: "env resolve": Command( target="nemo_gym.cli.env:dump_config", summary="Resolve the final config from configs, flags, and overrides.", + flags=(CONFIG,), ), "env packages": Command( target="nemo_gym.cli.env:pip_list", summary="Print pip packages for the selected resource server.", ), "env test": Command(target="nemo_gym.cli.env:test", summary="Test the resource server(s)."), - "env run": Command(target="nemo_gym.cli.env:run", summary="Start the servers."), + "env run": Command(target="nemo_gym.cli.env:run", summary="Start the servers.", flags=(CONFIG,)), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status."), "eval prepare": Command( target="nemo_gym.cli.eval:prepare_benchmark", summary="Prepare benchmark data and dump it to disk.", + flags=(CONFIG,), ), - "eval run": _toggle_command( - "no-serve", - "Collect against already-running servers instead of starting them.", - off_command="nemo_gym.cli.eval:e2e_rollout_collection", - on_command="nemo_gym.cli.eval:collect_rollouts", + "eval run": Command( + target=_eval_run, summary="Collate data, start servers, and collect rollouts.", + flags=(CONFIG, NO_SERVE), ), "eval aggregate": Command( target="nemo_gym.cli.eval:aggregate_rollouts", summary="Aggregate sharded rollout results.", + flags=(CONFIG,), ), "eval profile": Command( target="nemo_gym.cli.eval:reward_profile", @@ -157,10 +175,10 @@ def target(args: argparse.Namespace, overrides: list[str]) -> None: def _add_leaf(subparsers: argparse._SubParsersAction, name: str, command: Command) -> None: - leaf = subparsers.add_parser(name, parents=[COMMON], help=command.summary, description=command.summary) + leaf = subparsers.add_parser(name, help=command.summary, description=command.summary) leaf.set_defaults(_command=command) - if command.configure is not None: - command.configure(leaf) + for flag in command.flags: + flag.register(leaf) def build_parser() -> argparse.ArgumentParser: @@ -202,6 +220,7 @@ def main() -> None: args._parser.print_help() sys.exit(1) + overrides = [token for flag in command.flags for token in flag.translate_to_hydra(args)] + overrides if callable(command.target): command.target(args, overrides) else: From 82e35054e1fa22710b2d92ceba6be963da7f2644 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Tue, 16 Jun 2026 08:33:04 +0200 Subject: [PATCH 04/40] fix: don't pass unknown --flag or -f args to hydra Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index d8fd2ad925..2d75b2d743 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -211,6 +211,11 @@ def main() -> None: parser = build_parser() args, overrides = parser.parse_known_args() + # Hydra overrides never start with "-" so we treat them as unknown flags. + unknown_flags = [token for token in overrides if token.startswith("-")] + if unknown_flags: + getattr(args, "_parser", parser).error(f"unrecognized arguments: {' '.join(unknown_flags)}") + if args.version: dispatch(VERSION_TARGET, overrides) return From b075b9202b399eb32bc909a385a3d0ef97affd6b Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Tue, 16 Jun 2026 08:33:53 +0200 Subject: [PATCH 05/40] chore: add basic tests for testing new cli Signed-off-by: Marta Stepniewska-Dziubinska --- tests/unit_tests/test_cli_main.py | 110 ++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 tests/unit_tests/test_cli_main.py diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py new file mode 100644 index 0000000000..337a7f7079 --- /dev/null +++ b/tests/unit_tests/test_cli_main.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys + +import pytest +from pytest import MonkeyPatch + +import nemo_gym.cli.main as cli_main +from nemo_gym.cli.main import main + + +def _dispatch_for(monkeypatch: MonkeyPatch, argv: list[str]) -> tuple[str, list[str]]: + """Run the gym router for `argv` and return the (target, overrides) handed to dispatch.""" + captured: dict = {} + + def fake_dispatch(target: str, overrides: list[str]) -> None: + captured["target"] = target + captured["overrides"] = overrides + + monkeypatch.setattr(cli_main, "dispatch", fake_dispatch) + monkeypatch.setattr(sys, "argv", ["gym", *argv]) + main() + return captured["target"], captured["overrides"] + + +# `gym ` -> the legacy ng_ function it dispatches to, for the config-accepting commands. +CONFIG_COMMANDS = [ + (["env", "run"], "nemo_gym.cli.env:run"), + (["env", "resolve"], "nemo_gym.cli.env:dump_config"), + (["eval", "prepare"], "nemo_gym.cli.eval:prepare_benchmark"), + (["eval", "aggregate"], "nemo_gym.cli.eval:aggregate_rollouts"), + (["eval", "run"], "nemo_gym.cli.eval:e2e_rollout_collection"), + (["dataset", "collate"], "nemo_gym.cli.dataset:prepare_data"), +] + + +class TestConfigFlag: + @pytest.mark.parametrize("command, expected_target", CONFIG_COMMANDS) + def test_config_becomes_config_paths(self, monkeypatch: MonkeyPatch, command, expected_target) -> None: + """`gym --config X` dispatches to ng_ with +config_paths=[X].""" + target, overrides = _dispatch_for(monkeypatch, [*command, "--config", "my.yaml"]) + assert target == expected_target + assert overrides == ["+config_paths=[my.yaml]"] + + def test_repeated_config_joined_into_one_list(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--config", "a.yaml", "--config", "b.yaml"]) + + # We have this set of asserts to avoid asserting configs order in the string + assert len(overrides) == 1 + override = overrides[0] + assert override.startswith("+config_paths=[") + assert override.endswith("]") + assert "a.yaml" in override + assert "b.yaml" in override + + def test_config_is_prepended_before_passthrough_overrides(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--config", "a.yaml", "+foo=bar"]) + assert len(overrides) == 2 + assert "+config_paths=[a.yaml]" in overrides + assert "+foo=bar" in overrides + + def test_without_config_no_config_paths_added(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "run", "+foo=bar"]) + assert overrides == ["+foo=bar"] + + def test_config_rejected_on_non_config_command(self, monkeypatch: MonkeyPatch) -> None: + # `dataset rm` does not declare --config, so the router must reject it rather than leak it downstream. + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", "dataset", "rm", "--config", "x.yaml"]) + with pytest.raises(SystemExit): + main() + + +class TestStorageFlag: + @pytest.mark.parametrize( + "argv, expected_target", + [ + (["dataset", "upload"], "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli"), + (["dataset", "upload", "--storage", "hf"], "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli"), + (["dataset", "upload", "--storage", "gitlab"], "nemo_gym.cli.dataset:upload_jsonl_dataset_cli"), + (["dataset", "download"], "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli"), + (["dataset", "download", "--storage", "hf"], "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli"), + (["dataset", "download", "--storage", "gitlab"], "nemo_gym.cli.dataset:download_jsonl_dataset_cli"), + ], + ) + def test_storage_selects_backend(self, monkeypatch: MonkeyPatch, argv, expected_target) -> None: + target, _ = _dispatch_for(monkeypatch, argv) + assert target == expected_target + + def test_storage_does_not_leak_into_overrides(self, monkeypatch: MonkeyPatch) -> None: + # --storage only selects the target; it must not appear in the Hydra overrides. + _, overrides = _dispatch_for(monkeypatch, ["dataset", "upload", "--storage", "gitlab", "+foo=bar"]) + assert overrides == ["+foo=bar"] + + def test_invalid_storage_value_is_rejected(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr(sys, "argv", ["gym", "dataset", "upload", "--storage", "s3"]) + with pytest.raises(SystemExit): + main() From 95ddd2f69cf4f0c727d56f5dca07fa4717dd4fe6 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Tue, 16 Jun 2026 09:21:41 +0200 Subject: [PATCH 06/40] feat: run all tests if no resource server was passed Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 2d75b2d743..7a988d2d5e 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -75,12 +75,28 @@ def dispatch(target: str, overrides: list[str]) -> None: ) ) +ENTRYPOINT = Flag( + register=lambda p: p.add_argument( + "--resource_server", metavar="NAME", help="Name of the resource server to test. Tests all servers if omitted." + ), + translate_to_hydra=lambda args: ( + [f"+entrypoint=resources_servers/{args.resource_server}"] if args.resource_server else [] + ), +) + def _eval_run(args: argparse.Namespace, overrides: list[str]) -> None: target = "nemo_gym.cli.eval:collect_rollouts" if args.no_serve else "nemo_gym.cli.eval:e2e_rollout_collection" dispatch(target, overrides) +def _env_test(args: argparse.Namespace, overrides: list[str]) -> None: + # Run a single server's tests if one was named (--resource_server is translated to +entrypoint above) + # or an +entrypoint override was passed directly; otherwise run the whole suite. + has_entrypoint = any(override.lstrip("+").split("=", 1)[0] == "entrypoint" for override in overrides) + dispatch("nemo_gym.cli.env:test" if has_entrypoint else "nemo_gym.cli.env:test_all", overrides) + + def _dataset_upload(args: argparse.Namespace, overrides: list[str]) -> None: targets = { "hf": "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli", @@ -148,7 +164,11 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: target="nemo_gym.cli.env:pip_list", summary="Print pip packages for the selected resource server.", ), - "env test": Command(target="nemo_gym.cli.env:test", summary="Test the resource server(s)."), + "env test": Command( + target=_env_test, + summary="Test the resource server(s); runs all if no resource server is given.", + flags=(ENTRYPOINT,), + ), "env run": Command(target="nemo_gym.cli.env:run", summary="Start the servers.", flags=(CONFIG,)), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status."), "eval prepare": Command( From 0519f87452cbe8eace6e66001d7f2c76bf2ac134 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Tue, 16 Jun 2026 11:10:36 +0200 Subject: [PATCH 07/40] feat: add flags to eval run Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 79 ++++++++++++----- tests/unit_tests/test_cli_main.py | 135 ++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+), 22 deletions(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 7a988d2d5e..841e748275 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -49,6 +49,17 @@ def dispatch(target: str, overrides: list[str]) -> None: func() +def _value_flag(name: str, hydra_key: str, flag_help: str, *, aliases: tuple[str, ...] = ()) -> Flag: + """A `--name VALUE` flag that maps to the Hydra override `+=VALUE` (omitted when unset).""" + dest = name.replace("-", "_") + return Flag( + register=lambda p: p.add_argument(f"--{name}", *aliases, dest=dest, help=flag_help), + translate_to_hydra=lambda args: ( + [f"+{hydra_key}={getattr(args, dest)}"] if getattr(args, dest) is not None else [] + ), + ) + + # Shared flag: load Gym config files. Reused by every command that reads server/benchmark configs. CONFIG = Flag( register=lambda p: p.add_argument( @@ -60,30 +71,13 @@ def dispatch(target: str, overrides: list[str]) -> None: translate_to_hydra=lambda args: [f"+config_paths=[{','.join(args.config)}]"] if args.config else [], ) -# Command-specific flags read by the corresponding target callable below. -NO_SERVE = Flag( - register=lambda p: p.add_argument( - "--no-serve", - action="store_true", - help="Collect against already-running servers instead of starting them.", - ) -) - +# Shared flag: select the storage backend. Reused by `dataset upload` and `dataset download`. STORAGE = Flag( register=lambda p: p.add_argument( "--storage", choices=("hf", "gitlab"), default="hf", help="Storage backend (default: hf)." ) ) -ENTRYPOINT = Flag( - register=lambda p: p.add_argument( - "--resource_server", metavar="NAME", help="Name of the resource server to test. Tests all servers if omitted." - ), - translate_to_hydra=lambda args: ( - [f"+entrypoint=resources_servers/{args.resource_server}"] if args.resource_server else [] - ), -) - def _eval_run(args: argparse.Namespace, overrides: list[str]) -> None: target = "nemo_gym.cli.eval:collect_rollouts" if args.no_serve else "nemo_gym.cli.eval:e2e_rollout_collection" @@ -91,8 +85,9 @@ def _eval_run(args: argparse.Namespace, overrides: list[str]) -> None: def _env_test(args: argparse.Namespace, overrides: list[str]) -> None: - # Run a single server's tests if one was named (--resource_server is translated to +entrypoint above) - # or an +entrypoint override was passed directly; otherwise run the whole suite. + # Run a single server's tests if +entrypoint was passed. No need to check for + # --resource-server because it is translated to +entrypoint in the flag definition. + has_entrypoint = any(override.lstrip("+").split("=", 1)[0] == "entrypoint" for override in overrides) dispatch("nemo_gym.cli.env:test" if has_entrypoint else "nemo_gym.cli.env:test_all", overrides) @@ -167,7 +162,18 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "env test": Command( target=_env_test, summary="Test the resource server(s); runs all if no resource server is given.", - flags=(ENTRYPOINT,), + flags=( + Flag( + register=lambda p: p.add_argument( + "--resource-server", + metavar="NAME", + help="Name of the resource server to test. Tests all servers if omitted.", + ), + translate_to_hydra=lambda args: ( + [f"+entrypoint=resources_servers/{args.resource_server}"] if args.resource_server else [] + ), + ), + ), ), "env run": Command(target="nemo_gym.cli.env:run", summary="Start the servers.", flags=(CONFIG,)), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status."), @@ -179,7 +185,36 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "eval run": Command( target=_eval_run, summary="Collate data, start servers, and collect rollouts.", - flags=(CONFIG, NO_SERVE), + flags=( + CONFIG, + Flag( + register=lambda p: p.add_argument( + "--no-serve", + action="store_true", + help="Collect against already-running servers instead of starting them.", + ) + ), + Flag( + register=lambda p: p.add_argument( + "--resume", action="store_true", help="Resume from cached rollouts instead of recollecting." + ), + translate_to_hydra=lambda args: ["+resume_from_cache=true"] if args.resume else [], + ), + _value_flag("agent", "agent_name", "Agent to collect rollouts with.", aliases=("-a",)), + _value_flag("input", "input_jsonl_fpath", "Input tasks JSONL file.", aliases=("-i",)), + _value_flag("output", "output_jsonl_fpath", "Output rollouts JSONL file.", aliases=("-o",)), + _value_flag("limit", "limit", "Maximum number of tasks to run."), + _value_flag("num-repeats", "num_repeats", "Number of rollouts per task."), + _value_flag("prompt-config", "prompt_config", "Prompt template YAML to apply."), + _value_flag("concurrency", "num_samples_in_parallel", "Maximum number of concurrent samples."), + _value_flag("split", "split", "Dataset split to use (train, validation, or benchmark)."), + _value_flag("model-name", "policy_model_name", "Model name to evaluate."), + _value_flag("model-url", "policy_base_url", "Model server base URL."), + _value_flag("model-api-key", "policy_api_key", "Model server API key."), + _value_flag("temperature", "responses_create_params.temperature", "Sampling temperature."), + _value_flag("top-p", "responses_create_params.top_p", "Nucleus sampling top-p."), + _value_flag("max-output-tokens", "responses_create_params.max_output_tokens", "Maximum output tokens."), + ), ), "eval aggregate": Command( target="nemo_gym.cli.eval:aggregate_rollouts", diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 337a7f7079..5f91ad145b 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -108,3 +108,138 @@ def test_invalid_storage_value_is_rejected(self, monkeypatch: MonkeyPatch) -> No monkeypatch.setattr(sys, "argv", ["gym", "dataset", "upload", "--storage", "s3"]) with pytest.raises(SystemExit): main() + + +class TestEvalRunFlags: + @pytest.mark.parametrize( + "flag_argv, expected_override", + [ + (["--agent", "my_agent"], "+agent_name=my_agent"), + (["-a", "my_agent"], "+agent_name=my_agent"), + (["--input", "in.jsonl"], "+input_jsonl_fpath=in.jsonl"), + (["-i", "in.jsonl"], "+input_jsonl_fpath=in.jsonl"), + (["--output", "out.jsonl"], "+output_jsonl_fpath=out.jsonl"), + (["-o", "out.jsonl"], "+output_jsonl_fpath=out.jsonl"), + (["--limit", "1024"], "+limit=1024"), + (["--num-repeats", "4"], "+num_repeats=4"), + (["--concurrency", "10"], "+num_samples_in_parallel=10"), + (["--prompt-config", "p.yaml"], "+prompt_config=p.yaml"), + (["--split", "benchmark"], "+split=benchmark"), + (["--model-name", "openai/gpt-oss-120b"], "+policy_model_name=openai/gpt-oss-120b"), + (["--model-url", "http://0.0.0.0:10240/v1"], "+policy_base_url=http://0.0.0.0:10240/v1"), + (["--model-api-key", "sk-your-api-key"], "+policy_api_key=sk-your-api-key"), + (["--temperature", "1.0"], "+responses_create_params.temperature=1.0"), + (["--top-p", "1.0"], "+responses_create_params.top_p=1.0"), + (["--max-output-tokens", "4096"], "+responses_create_params.max_output_tokens=4096"), + (["--resume"], "+resume_from_cache=true"), + ], + ) + def test_flag_maps_to_single_override(self, monkeypatch: MonkeyPatch, flag_argv, expected_override) -> None: + _, overrides = _dispatch_for(monkeypatch, ["eval", "run", *flag_argv]) + assert overrides == [expected_override] + + def test_unset_flags_contribute_nothing(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--agent", "x"]) + assert overrides == ["+agent_name=x"] + + def test_default_dispatches_e2e(self, monkeypatch: MonkeyPatch) -> None: + target, _ = _dispatch_for(monkeypatch, ["eval", "run"]) + assert target == "nemo_gym.cli.eval:e2e_rollout_collection" + + def test_no_serve_dispatches_collect_without_override(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--no-serve"]) + assert target == "nemo_gym.cli.eval:collect_rollouts" + assert overrides == [] + + def test_readme_collect_rollouts_example(self, monkeypatch: MonkeyPatch) -> None: + # From resources_servers/my_weather_tool README: + # ng_collect_rollouts +agent_name=... +input_jsonl_fpath=... +output_jsonl_fpath=... +limit=1024 +num_repeats=1 + target, overrides = _dispatch_for( + monkeypatch, + [ + "eval", + "run", + "--no-serve", + "--agent", + "my_weather_tool_simple_agent", + "--input", + "resources_servers/my_weather_tool/data/example.jsonl", + "--output", + "resources_servers/my_weather_tool/data/example_rollouts.jsonl", + "--limit", + "1024", + "--num-repeats", + "1", + ], + ) + assert target == "nemo_gym.cli.eval:collect_rollouts" + assert set(overrides) == { + "+agent_name=my_weather_tool_simple_agent", + "+input_jsonl_fpath=resources_servers/my_weather_tool/data/example.jsonl", + "+output_jsonl_fpath=resources_servers/my_weather_tool/data/example_rollouts.jsonl", + "+limit=1024", + "+num_repeats=1", + } + + def test_readme_model_and_sampling_example(self, monkeypatch: MonkeyPatch) -> None: + # From the gpt-oss eval example: ++policy_* and ++responses_create_params.* overrides. + _, overrides = _dispatch_for( + monkeypatch, + [ + "eval", + "run", + "--model-name", + "openai/gpt-oss-120b", + "--model-url", + "http://0.0.0.0:10240/v1", + "--model-api-key", + "dummy_key", + "--temperature", + "1.0", + "--top-p", + "1.0", + ], + ) + assert set(overrides) == { + "+policy_model_name=openai/gpt-oss-120b", + "+policy_base_url=http://0.0.0.0:10240/v1", + "+policy_api_key=dummy_key", + "+responses_create_params.temperature=1.0", + "+responses_create_params.top_p=1.0", + } + + def test_flags_compose_with_config_and_passthrough(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + [ + "eval", + "run", + "--no-serve", + "--config", + "b.yaml", + "--agent", + "a", + "+responses_create_params.tool_choice=auto", + ], + ) + assert target == "nemo_gym.cli.eval:collect_rollouts" + assert "+config_paths=[b.yaml]" in overrides + assert "+agent_name=a" in overrides + assert "+responses_create_params.tool_choice=auto" in overrides # unknown +override passes through + + +class TestEnvTestResourceServerFlag: + def test_no_resource_server_runs_all(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["env", "test"]) + assert target == "nemo_gym.cli.env:test_all" + assert overrides == [] + + def test_resource_server_name_translates_to_entrypoint(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["env", "test", "--resource-server", "gpqa"]) + assert target == "nemo_gym.cli.env:test" + assert overrides == ["+entrypoint=resources_servers/gpqa"] + + def test_direct_entrypoint_override_also_runs_single(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["env", "test", "+entrypoint=resources_servers/gpqa"]) + assert target == "nemo_gym.cli.env:test" + assert overrides == ["+entrypoint=resources_servers/gpqa"] From e0135254ebe843e00bc92e519c024d5640c8f535 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Tue, 16 Jun 2026 13:39:15 +0200 Subject: [PATCH 08/40] feat: add flags to dataset commands Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 77 ++++++++++++--- tests/unit_tests/test_cli_main.py | 150 ++++++++++++++++++++++++++++++ 2 files changed, 216 insertions(+), 11 deletions(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 841e748275..df303a5262 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -49,17 +49,28 @@ def dispatch(target: str, overrides: list[str]) -> None: func() -def _value_flag(name: str, hydra_key: str, flag_help: str, *, aliases: tuple[str, ...] = ()) -> Flag: +def _value_flag( + name: str, hydra_key: str, flag_help: str, *, aliases: tuple[str, ...] = (), choices: tuple[str, ...] | None = None +) -> Flag: """A `--name VALUE` flag that maps to the Hydra override `+=VALUE` (omitted when unset).""" dest = name.replace("-", "_") return Flag( - register=lambda p: p.add_argument(f"--{name}", *aliases, dest=dest, help=flag_help), + register=lambda p: p.add_argument(f"--{name}", *aliases, dest=dest, choices=choices, help=flag_help), translate_to_hydra=lambda args: ( [f"+{hydra_key}={getattr(args, dest)}"] if getattr(args, dest) is not None else [] ), ) +def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: + """A `--name` store_true flag that maps to the Hydra override `+=true` when set.""" + dest = name.replace("-", "_") + return Flag( + register=lambda p: p.add_argument(f"--{name}", action="store_true", help=flag_help), + translate_to_hydra=lambda args: [f"+{hydra_key}=true"] if getattr(args, dest) else [], + ) + + # Shared flag: load Gym config files. Reused by every command that reads server/benchmark configs. CONFIG = Flag( register=lambda p: p.add_argument( @@ -122,29 +133,78 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "dataset upload": Command( target=_dataset_upload, summary="Upload a prepared dataset to HF (default) or GitLab.", - flags=(STORAGE,), + flags=( + STORAGE, + _value_flag("input", "input_jsonl_fpath", "Local JSONL file to upload.", aliases=("-i",)), + _value_flag("name", "dataset_name", "Dataset name."), + # GitLab stores it as `version`, HF as `revision`; emit both and let each backend keep its own. + Flag( + register=lambda p: p.add_argument( + "--revision", dest="revision", help="Dataset revision (version) to download." + ), + translate_to_hydra=lambda args: ( + # we set both version and revision because GitLab and HF use different keys + # and we extra="ignore" so it's safe to set both + [f"+version={args.revision}", f"+revision={args.revision}"] if args.revision is not None else [] + ), + ), + _value_flag("split", "split", "Dataset split (HF only)."), + _bool_flag("create-pr", "create_pr", "Open a pull request instead of committing directly (HF only)."), + ), ), "dataset download": Command( target=_dataset_download, summary="Download a dataset from HF (default) or GitLab.", - flags=(STORAGE,), + flags=( + STORAGE, + _value_flag("repo-id", "repo_id", "HF repo id, e.g. org/dataset (HF only)."), + _value_flag("name", "dataset_name", "Dataset name (GitLab only)."), + # NOTE(martas): HF download does not allow to specify revision + _value_flag("revision", "version", "Dataset version (GitLab only)."), + _value_flag( + "artifact", "artifact_fpath", "Remote file to fetch (GitLab: required; HF: optional raw file)." + ), + _value_flag("output", "output_fpath", "Local destination file.", aliases=("-o",)), + _value_flag( + "output-dir", "output_dirpath", "Local destination directory; needed for all splits (HF only)." + ), + _value_flag("split", "split", "Dataset split (HF only)."), + ), ), "dataset rm": Command( target="nemo_gym.cli.dataset:delete_jsonl_dataset_from_gitlab_cli", summary="Delete a dataset from GitLab.", + flags=(_value_flag("name", "dataset_name", "Name of the dataset to delete."),), ), "dataset migrate": Command( target="nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli", summary="Transfer a dataset from GitLab to HF.", + flags=( + _value_flag("input", "input_jsonl_fpath", "Local JSONL file to upload to HF.", aliases=("-i",)), + _value_flag("name", "dataset_name", "Dataset name."), + _value_flag("revision", "revision", "Dataset revision (HF)."), + _value_flag("split", "split", "Dataset split."), + _bool_flag("create-pr", "create_pr", "Open a pull request instead of committing directly."), + ), ), "dataset render": Command( target="nemo_gym.cli.dataset:materialize_prompts_cli", summary="Generate a dataset preview.", + flags=( + _value_flag("input", "input_jsonl_fpath", "Raw input JSONL file.", aliases=("-i",)), + _value_flag("prompt-config", "prompt_config", "Prompt template YAML to apply."), + _value_flag("output", "output_jsonl_fpath", "Output JSONL file.", aliases=("-o",)), + ), ), "dataset collate": Command( target="nemo_gym.cli.dataset:prepare_data", summary="Validate and collate the dataset.", - flags=(CONFIG,), + flags=( + CONFIG, + _value_flag("mode", "mode", "Data preparation mode.", choices=("train_preparation", "example_validation")), + _value_flag("output-dir", "output_dirpath", "Output directory for the prepared data."), + _bool_flag("download", "should_download", "Download source datasets before collating."), + ), ), "env init": Command( target="nemo_gym.cli.env:init_resources_server", @@ -194,12 +254,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: help="Collect against already-running servers instead of starting them.", ) ), - Flag( - register=lambda p: p.add_argument( - "--resume", action="store_true", help="Resume from cached rollouts instead of recollecting." - ), - translate_to_hydra=lambda args: ["+resume_from_cache=true"] if args.resume else [], - ), + _bool_flag("resume", "resume_from_cache", "Resume from cached rollouts instead of recollecting."), _value_flag("agent", "agent_name", "Agent to collect rollouts with.", aliases=("-a",)), _value_flag("input", "input_jsonl_fpath", "Input tasks JSONL file.", aliases=("-i",)), _value_flag("output", "output_jsonl_fpath", "Output rollouts JSONL file.", aliases=("-o",)), diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 5f91ad145b..f5ef310c1f 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -243,3 +243,153 @@ def test_direct_entrypoint_override_also_runs_single(self, monkeypatch: MonkeyPa target, overrides = _dispatch_for(monkeypatch, ["env", "test", "+entrypoint=resources_servers/gpqa"]) assert target == "nemo_gym.cli.env:test" assert overrides == ["+entrypoint=resources_servers/gpqa"] + + +class TestDatasetFlags: + def test_upload_hf_default(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + ["dataset", "upload", "-i", "data/train.jsonl", "--name", "my_ds", "--split", "train", "--create-pr"], + ) + assert target == "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli" + assert set(overrides) == { + "+input_jsonl_fpath=data/train.jsonl", + "+dataset_name=my_ds", + "+split=train", + "+create_pr=true", + } + + def test_upload_gitlab(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + [ + "dataset", + "upload", + "--storage", + "gitlab", + "-i", + "data/train.jsonl", + "--name", + "my_ds", + "--revision", + "0.0.1", + ], + ) + assert target == "nemo_gym.cli.dataset:upload_jsonl_dataset_cli" + overrides.remove( + "+revision=0.0.1" + ) # we set both version and revision because GitLab and HF use different keys + assert set(overrides) == { + "+input_jsonl_fpath=data/train.jsonl", + "+dataset_name=my_ds", + "+version=0.0.1", + } + + def test_download_hf_default(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + [ + "dataset", + "download", + "--repo-id", + "org/my_ds", + "--artifact", + "train.jsonl", + "--output-dir", + "./data", + "--split", + "train", + ], + ) + assert target == "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli" + assert set(overrides) == { + "+repo_id=org/my_ds", + "+artifact_fpath=train.jsonl", + "+output_dirpath=./data", + "+split=train", + } + + def test_download_gitlab(self, monkeypatch: MonkeyPatch) -> None: + # On download, --revision is GitLab-only and maps to +version (HF download has no revision field). + target, overrides = _dispatch_for( + monkeypatch, + [ + "dataset", + "download", + "--storage", + "gitlab", + "--name", + "my_ds", + "--revision", + "0.0.1", + "--artifact", + "train.jsonl", + "-o", + "./train.jsonl", + ], + ) + assert target == "nemo_gym.cli.dataset:download_jsonl_dataset_cli" + assert set(overrides) == { + "+dataset_name=my_ds", + "+version=0.0.1", + "+artifact_fpath=train.jsonl", + "+output_fpath=./train.jsonl", + } + + def test_rm(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["dataset", "rm", "--name", "my_ds"]) + assert target == "nemo_gym.cli.dataset:delete_jsonl_dataset_from_gitlab_cli" + assert overrides == ["+dataset_name=my_ds"] + + def test_migrate_revision_maps_to_hf_revision(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + ["dataset", "migrate", "-i", "data/train.jsonl", "--name", "my_ds", "--revision", "r1", "--create-pr"], + ) + assert target == "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli" + assert set(overrides) == { + "+input_jsonl_fpath=data/train.jsonl", + "+dataset_name=my_ds", + "+revision=r1", + "+create_pr=true", + } + + def test_render(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, ["dataset", "render", "-i", "raw.jsonl", "--prompt-config", "p.yaml", "-o", "out.jsonl"] + ) + assert target == "nemo_gym.cli.dataset:materialize_prompts_cli" + assert overrides == ["+input_jsonl_fpath=raw.jsonl", "+prompt_config=p.yaml", "+output_jsonl_fpath=out.jsonl"] + + def test_collate(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + [ + "dataset", + "collate", + "--config", + "c.yaml", + "--mode", + "train_preparation", + "--output-dir", + "./prep", + "--download", + ], + ) + assert target == "nemo_gym.cli.dataset:prepare_data" + assert set(overrides) == { + "+config_paths=[c.yaml]", + "+mode=train_preparation", + "+output_dirpath=./prep", + "+should_download=true", + } + + def test_bool_flags_omitted_when_unset(self, monkeypatch: MonkeyPatch) -> None: + # --create-pr not passed -> no +create_pr override leaks in. + _, overrides = _dispatch_for(monkeypatch, ["dataset", "upload", "--name", "my_ds"]) + assert overrides == ["+dataset_name=my_ds"] + + def test_collate_mode_rejects_invalid_choice(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr(sys, "argv", ["gym", "dataset", "collate", "--mode", "bogus"]) + with pytest.raises(SystemExit): + main() From 6a7925989f1ff233b60819359680085853348a53 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Tue, 16 Jun 2026 14:13:17 +0200 Subject: [PATCH 09/40] feat: add flags for other eval commands Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 16 +++++++++++++++- tests/unit_tests/test_cli_main.py | 26 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index df303a5262..f249f82388 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -274,11 +274,25 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "eval aggregate": Command( target="nemo_gym.cli.eval:aggregate_rollouts", summary="Aggregate sharded rollout results.", - flags=(CONFIG,), + flags=( + CONFIG, + _value_flag( + "output", + "output_jsonl_fpath", + "Path for the merged rollouts and aggregate-metrics file.", + aliases=("-o",), + ), + ), ), "eval profile": Command( target="nemo_gym.cli.eval:reward_profile", summary="Compute a reward profile from rollouts.", + flags=( + _value_flag( + "inputs", "materialized_inputs_jsonl_fpath", "Materialized inputs JSONL fed to rollout collection." + ), + _value_flag("rollouts", "rollouts_jsonl_fpath", "Rollouts JSONL produced by collection."), + ), ), "dev test": Command(target="nemo_gym.cli.dev:dev_test", summary="Run NeMo Gym's unit tests."), } diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index f5ef310c1f..c37353ca42 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -393,3 +393,29 @@ def test_collate_mode_rejects_invalid_choice(self, monkeypatch: MonkeyPatch) -> monkeypatch.setattr(sys, "argv", ["gym", "dataset", "collate", "--mode", "bogus"]) with pytest.raises(SystemExit): main() + + +class TestEvalAggregateFlags: + def test_output_flag(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["eval", "aggregate", "-o", "out.jsonl"]) + assert target == "nemo_gym.cli.eval:aggregate_rollouts" + assert overrides == ["+output_jsonl_fpath=out.jsonl"] + + +class TestEvalProfileFlags: + def test_profile_flags(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, ["eval", "profile", "--inputs", "in.jsonl", "--rollouts", "r.jsonl"] + ) + assert target == "nemo_gym.cli.eval:reward_profile" + assert set(overrides) == { + "+materialized_inputs_jsonl_fpath=in.jsonl", + "+rollouts_jsonl_fpath=r.jsonl", + } + + def test_profile_does_not_accept_config(self, monkeypatch: MonkeyPatch) -> None: + # reward_profile reads file paths, not config_paths, so --config is not offered and is rejected. + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", "eval", "profile", "--config", "x.yaml"]) + with pytest.raises(SystemExit): + main() From bed657663c5cfd80298d3981d22c75d4a326c9e3 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Tue, 16 Jun 2026 14:43:03 +0200 Subject: [PATCH 10/40] feat: add flags for env commands Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 43 +++++++++++++++++++------------ tests/unit_tests/test_cli_main.py | 43 +++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 16 deletions(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index f249f82388..47e8d2efcd 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -89,6 +89,19 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: ) ) +# Shared model-server flags. Reused by commands that spin up / target a model server (`eval run`, `env run`). +MODEL_NAME = _value_flag("model-name", "policy_model_name", "Model name.") +MODEL_URL = _value_flag("model-url", "policy_base_url", "Model server base URL.") +MODEL_API_KEY = _value_flag("model-api-key", "policy_api_key", "Model server API key.") + +# Shared flag: select a single resource server by name. Reused by `env test`, `env init`, and `env packages`. +RESOURCE_SERVER = Flag( + register=lambda p: p.add_argument("--resource-server", metavar="NAME", help="Name of the resource server."), + translate_to_hydra=lambda args: ( + [f"+entrypoint=resources_servers/{args.resource_server}"] if args.resource_server else [] + ), +) + def _eval_run(args: argparse.Namespace, overrides: list[str]) -> None: target = "nemo_gym.cli.eval:collect_rollouts" if args.no_serve else "nemo_gym.cli.eval:e2e_rollout_collection" @@ -209,6 +222,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "env init": Command( target="nemo_gym.cli.env:init_resources_server", summary="Scaffold config for a new server, benchmark, or agent.", + flags=(RESOURCE_SERVER,), ), "env resolve": Command( target="nemo_gym.cli.env:dump_config", @@ -218,24 +232,21 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "env packages": Command( target="nemo_gym.cli.env:pip_list", summary="Print pip packages for the selected resource server.", + flags=( + RESOURCE_SERVER, + _bool_flag("outdated", "outdated", "List only outdated packages."), + ), ), "env test": Command( target=_env_test, summary="Test the resource server(s); runs all if no resource server is given.", - flags=( - Flag( - register=lambda p: p.add_argument( - "--resource-server", - metavar="NAME", - help="Name of the resource server to test. Tests all servers if omitted.", - ), - translate_to_hydra=lambda args: ( - [f"+entrypoint=resources_servers/{args.resource_server}"] if args.resource_server else [] - ), - ), - ), + flags=(RESOURCE_SERVER,), + ), + "env run": Command( + target="nemo_gym.cli.env:run", + summary="Start the servers.", + flags=(CONFIG, MODEL_NAME, MODEL_URL, MODEL_API_KEY), ), - "env run": Command(target="nemo_gym.cli.env:run", summary="Start the servers.", flags=(CONFIG,)), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status."), "eval prepare": Command( target="nemo_gym.cli.eval:prepare_benchmark", @@ -263,9 +274,9 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: _value_flag("prompt-config", "prompt_config", "Prompt template YAML to apply."), _value_flag("concurrency", "num_samples_in_parallel", "Maximum number of concurrent samples."), _value_flag("split", "split", "Dataset split to use (train, validation, or benchmark)."), - _value_flag("model-name", "policy_model_name", "Model name to evaluate."), - _value_flag("model-url", "policy_base_url", "Model server base URL."), - _value_flag("model-api-key", "policy_api_key", "Model server API key."), + MODEL_NAME, + MODEL_URL, + MODEL_API_KEY, _value_flag("temperature", "responses_create_params.temperature", "Sampling temperature."), _value_flag("top-p", "responses_create_params.top_p", "Nucleus sampling top-p."), _value_flag("max-output-tokens", "responses_create_params.max_output_tokens", "Maximum output tokens."), diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index c37353ca42..59f3a8f567 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -419,3 +419,46 @@ def test_profile_does_not_accept_config(self, monkeypatch: MonkeyPatch) -> None: monkeypatch.setattr(sys, "argv", ["gym", "eval", "profile", "--config", "x.yaml"]) with pytest.raises(SystemExit): main() + + +class TestEnvRunFlags: + def test_model_flags(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + [ + "env", + "run", + "--config", + "c.yaml", + "--model-name", + "gpt", + "--model-url", + "http://x", + "--model-api-key", + "k", + ], + ) + assert target == "nemo_gym.cli.env:run" + assert set(overrides) == { + "+config_paths=[c.yaml]", + "+policy_model_name=gpt", + "+policy_base_url=http://x", + "+policy_api_key=k", + } + + +class TestEnvInitFlags: + def test_resource_server_translates_to_entrypoint(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["env", "init", "--resource-server", "my_server"]) + assert target == "nemo_gym.cli.env:init_resources_server" + assert overrides == ["+entrypoint=resources_servers/my_server"] + + +class TestEnvPackagesFlags: + def test_flags(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["env", "packages", "--resource-server", "gpqa", "--outdated"]) + assert target == "nemo_gym.cli.env:pip_list" + assert set(overrides) == { + "+entrypoint=resources_servers/gpqa", + "+outdated=true", + } From 4a053711bc5815c60bd17b3f26aadca187089832 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 17 Jun 2026 10:58:16 +0200 Subject: [PATCH 11/40] chore: print deprecation notice for all legacy commands Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/legacy.py | 79 +++++++++++++++++++++++++++ pyproject.toml | 100 +++++++++++++++++------------------ tests/unit_tests/test_cli.py | 28 +++------- 3 files changed, 135 insertions(+), 72 deletions(-) create mode 100644 nemo_gym/cli/legacy.py diff --git a/nemo_gym/cli/legacy.py b/nemo_gym/cli/legacy.py new file mode 100644 index 0000000000..837f4b52ba --- /dev/null +++ b/nemo_gym/cli/legacy.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Backward-compatibility shim for the legacy ``ng_*`` / ``nemo_gym_*`` commands. + +Every legacy console script points here; the script name (``sys.argv[0]``) identifies which +command was invoked. We print a one-time deprecation notice mapping it to the new ``gym`` +command, then re-enter the ``gym`` router so the command keeps working (REQ 7). +""" + +import sys +from pathlib import Path + +from nemo_gym.cli.main import dispatch +from nemo_gym.cli.main import main as gym_main + + +# Legacy command (ng_/nemo_gym_ prefix stripped) -> equivalent `gym` subcommand tokens. +LEGACY = { + "run": ["env", "run"], + "test": ["env", "test"], + "test_all": ["env", "test"], + "dev_test": ["dev", "test"], + "init_resources_server": ["env", "init"], + "list_benchmarks": ["list", "benchmarks"], + "prepare_benchmark": ["eval", "prepare"], + "collect_rollouts": ["eval", "run", "--no-serve"], + "e2e_collect_rollouts": ["eval", "run"], + "aggregate_rollouts": ["eval", "aggregate"], + "materialize_prompts": ["dataset", "render"], + "reward_profile": ["eval", "profile"], + "upload_dataset_to_gitlab": ["dataset", "upload", "--storage", "gitlab"], + "download_dataset_from_gitlab": ["dataset", "download", "--storage", "gitlab"], + "prepare_data": ["dataset", "collate"], + "upload_dataset_to_hf": ["dataset", "upload"], + "download_dataset_from_hf": ["dataset", "download"], + "gitlab_to_hf_dataset": ["dataset", "migrate"], + "delete_dataset_from_gitlab": ["dataset", "rm"], + "dump_config": ["env", "resolve"], + "help": ["--help"], + "status": ["env", "status"], + "pip_list": ["env", "packages"], + "version": ["--version"], +} + + +def main() -> None: + alias = Path(sys.argv[0]).name + key = alias.removeprefix("nemo_gym_").removeprefix("ng_") + + # `reinstall` has no `gym` equivalent (`gym install` was dropped); point users at the uv command it runs. + if key == "reinstall": + print( + f"⚠ `{alias}` is deprecated and will be removed in a future release; " + f"run `uv sync --extra dev --group docs` instead.", + file=sys.stderr, + ) + dispatch("nemo_gym.cli.general:reinstall", sys.argv[1:]) + return + + tokens = LEGACY[key] + print( + f"⚠ `{alias}` is deprecated and will be removed in a future release; use `gym {' '.join(tokens)}` instead.", + file=sys.stderr, + ) + # Re-enter the gym router with the equivalent subcommand, preserving the user's Hydra overrides. + sys.argv = [sys.argv[0], *tokens, *sys.argv[1:]] + gym_main() diff --git a/pyproject.toml b/pyproject.toml index 5196613221..6c576f2de8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -318,86 +318,86 @@ ng = "nemo_gym.cli.main:main" ########################## DEPRECATED COMMANDS ########################## # Run/test scripts for servers. -nemo_gym_run = "nemo_gym.cli.env:run" -ng_run = "nemo_gym.cli.env:run" -nemo_gym_test = "nemo_gym.cli.env:test" -ng_test = "nemo_gym.cli.env:test" -nemo_gym_test_all = "nemo_gym.cli.env:test_all" -ng_test_all = "nemo_gym.cli.env:test_all" +nemo_gym_run = "nemo_gym.cli.legacy:main" +ng_run = "nemo_gym.cli.legacy:main" +nemo_gym_test = "nemo_gym.cli.legacy:main" +ng_test = "nemo_gym.cli.legacy:main" +nemo_gym_test_all = "nemo_gym.cli.legacy:main" +ng_test_all = "nemo_gym.cli.legacy:main" # Development convenience aliases. -nemo_gym_dev_test = "nemo_gym.cli.dev:dev_test" -ng_dev_test = "nemo_gym.cli.dev:dev_test" -nemo_gym_init_resources_server = "nemo_gym.cli.env:init_resources_server" -ng_init_resources_server = "nemo_gym.cli.env:init_resources_server" +nemo_gym_dev_test = "nemo_gym.cli.legacy:main" +ng_dev_test = "nemo_gym.cli.legacy:main" +nemo_gym_init_resources_server = "nemo_gym.cli.legacy:main" +ng_init_resources_server = "nemo_gym.cli.legacy:main" # Benchmarks -nemo_gym_list_benchmarks = "nemo_gym.cli.eval:list_benchmarks" -ng_list_benchmarks = "nemo_gym.cli.eval:list_benchmarks" -nemo_gym_prepare_benchmark = "nemo_gym.cli.eval:prepare_benchmark" -ng_prepare_benchmark = "nemo_gym.cli.eval:prepare_benchmark" +nemo_gym_list_benchmarks = "nemo_gym.cli.legacy:main" +ng_list_benchmarks = "nemo_gym.cli.legacy:main" +nemo_gym_prepare_benchmark = "nemo_gym.cli.legacy:main" +ng_prepare_benchmark = "nemo_gym.cli.legacy:main" # Rollout collection -nemo_gym_collect_rollouts = "nemo_gym.cli.eval:collect_rollouts" -ng_collect_rollouts = "nemo_gym.cli.eval:collect_rollouts" -nemo_gym_e2e_collect_rollouts = "nemo_gym.cli.eval:e2e_rollout_collection" -ng_e2e_collect_rollouts = "nemo_gym.cli.eval:e2e_rollout_collection" -nemo_gym_aggregate_rollouts = "nemo_gym.cli.eval:aggregate_rollouts" -ng_aggregate_rollouts = "nemo_gym.cli.eval:aggregate_rollouts" +nemo_gym_collect_rollouts = "nemo_gym.cli.legacy:main" +ng_collect_rollouts = "nemo_gym.cli.legacy:main" +nemo_gym_e2e_collect_rollouts = "nemo_gym.cli.legacy:main" +ng_e2e_collect_rollouts = "nemo_gym.cli.legacy:main" +nemo_gym_aggregate_rollouts = "nemo_gym.cli.legacy:main" +ng_aggregate_rollouts = "nemo_gym.cli.legacy:main" # Prompt materialization -nemo_gym_materialize_prompts = "nemo_gym.cli.dataset:materialize_prompts_cli" -ng_materialize_prompts = "nemo_gym.cli.dataset:materialize_prompts_cli" +nemo_gym_materialize_prompts = "nemo_gym.cli.legacy:main" +ng_materialize_prompts = "nemo_gym.cli.legacy:main" # Reward profiling -nemo_gym_reward_profile = "nemo_gym.cli.eval:reward_profile" -ng_reward_profile = "nemo_gym.cli.eval:reward_profile" +nemo_gym_reward_profile = "nemo_gym.cli.legacy:main" +ng_reward_profile = "nemo_gym.cli.legacy:main" # Dataset management -nemo_gym_upload_dataset_to_gitlab = "nemo_gym.cli.dataset:upload_jsonl_dataset_cli" -ng_upload_dataset_to_gitlab = "nemo_gym.cli.dataset:upload_jsonl_dataset_cli" -nemo_gym_download_dataset_from_gitlab = "nemo_gym.cli.dataset:download_jsonl_dataset_cli" -ng_download_dataset_from_gitlab = "nemo_gym.cli.dataset:download_jsonl_dataset_cli" -nemo_gym_prepare_data = "nemo_gym.cli.dataset:prepare_data" -ng_prepare_data = "nemo_gym.cli.dataset:prepare_data" +nemo_gym_upload_dataset_to_gitlab = "nemo_gym.cli.legacy:main" +ng_upload_dataset_to_gitlab = "nemo_gym.cli.legacy:main" +nemo_gym_download_dataset_from_gitlab = "nemo_gym.cli.legacy:main" +ng_download_dataset_from_gitlab = "nemo_gym.cli.legacy:main" +nemo_gym_prepare_data = "nemo_gym.cli.legacy:main" +ng_prepare_data = "nemo_gym.cli.legacy:main" # HF dataset upload and download -nemo_gym_upload_dataset_to_hf = "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli" -ng_upload_dataset_to_hf = "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli" -nemo_gym_download_dataset_from_hf = "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli" -ng_download_dataset_from_hf = "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli" +nemo_gym_upload_dataset_to_hf = "nemo_gym.cli.legacy:main" +ng_upload_dataset_to_hf = "nemo_gym.cli.legacy:main" +nemo_gym_download_dataset_from_hf = "nemo_gym.cli.legacy:main" +ng_download_dataset_from_hf = "nemo_gym.cli.legacy:main" # Gitlab -> HF (upload then delete) -nemo_gym_gitlab_to_hf_dataset = "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli" -ng_gitlab_to_hf_dataset = "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli" +nemo_gym_gitlab_to_hf_dataset = "nemo_gym.cli.legacy:main" +ng_gitlab_to_hf_dataset = "nemo_gym.cli.legacy:main" # Manually delete from Gitlab -nemo_gym_delete_dataset_from_gitlab = "nemo_gym.cli.dataset:delete_jsonl_dataset_from_gitlab_cli" -ng_delete_dataset_from_gitlab = "nemo_gym.cli.dataset:delete_jsonl_dataset_from_gitlab_cli" +nemo_gym_delete_dataset_from_gitlab = "nemo_gym.cli.legacy:main" +ng_delete_dataset_from_gitlab = "nemo_gym.cli.legacy:main" # Configuration utils -nemo_gym_dump_config = "nemo_gym.cli.env:dump_config" -ng_dump_config = "nemo_gym.cli.env:dump_config" +nemo_gym_dump_config = "nemo_gym.cli.legacy:main" +ng_dump_config = "nemo_gym.cli.legacy:main" # Display help -nemo_gym_help = "nemo_gym.cli.general:display_help_legacy" -ng_help = "nemo_gym.cli.general:display_help_legacy" +nemo_gym_help = "nemo_gym.cli.legacy:main" +ng_help = "nemo_gym.cli.legacy:main" # Server status -nemo_gym_status = "nemo_gym.cli.env:status" -ng_status = "nemo_gym.cli.env:status" +nemo_gym_status = "nemo_gym.cli.legacy:main" +ng_status = "nemo_gym.cli.legacy:main" # Environment-specific uv pip list -nemo_gym_pip_list = "nemo_gym.cli.env:pip_list" -ng_pip_list = "nemo_gym.cli.env:pip_list" +nemo_gym_pip_list = "nemo_gym.cli.legacy:main" +ng_pip_list = "nemo_gym.cli.legacy:main" # Display version -nemo_gym_version = "nemo_gym.cli.general:version" -ng_version = "nemo_gym.cli.general:version" +nemo_gym_version = "nemo_gym.cli.legacy:main" +ng_version = "nemo_gym.cli.legacy:main" # Re-install Gym and dependencies -nemo_gym_reinstall = "nemo_gym.cli.general:reinstall" -ng_reinstall = "nemo_gym.cli.general:reinstall" +nemo_gym_reinstall = "nemo_gym.cli.legacy:main" +ng_reinstall = "nemo_gym.cli.legacy:main" [tool.setuptools.packages.find] where = ["."] diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index 91106ec850..cc6c5c9772 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -22,7 +22,7 @@ from unittest.mock import MagicMock, patch from omegaconf import OmegaConf -from pytest import MonkeyPatch, raises +from pytest import MonkeyPatch import nemo_gym.global_config from nemo_gym import PARENT_DIR @@ -42,32 +42,16 @@ class TestCLI: def test_sanity(self) -> None: RunConfig(entrypoint="", name="") - def test_pyproject_scripts(self) -> None: + def test_pyproject_scripts_are_importable(self) -> None: + """Every console-script entry point must resolve to an importable callable.""" pyproject_path = PARENT_DIR / "pyproject.toml" with pyproject_path.open("rb") as f: pyproject_data = tomllib.load(f) - project_scripts = pyproject_data["project"]["scripts"] - - for script_name, import_path in project_scripts.items(): - # Dedupe `nemo_gym_*` from `ng_*` commands - if not script_name.startswith("ng_"): - continue - - # We only test `+h=true` and not `+help=true` - print(f"Running `{script_name} +h=true`") - + for script_name, import_path in pyproject_data["project"]["scripts"].items(): module, fn = import_path.split(":") - fn = getattr(import_module(module), fn) - - with MonkeyPatch.context() as mp: - mp.setattr(nemo_gym.global_config, "_GLOBAL_CONFIG_DICT", OmegaConf.create({"h": True})) - - text_trap = StringIO() - mp.setattr(sys, "stdout", text_trap) - - with raises(SystemExit): - fn() + target = getattr(import_module(module), fn) + assert callable(target), f"{script_name} -> {import_path} is not callable" def test_display_help_discovers_scripts(self) -> None: with MonkeyPatch.context() as mp: From 17b79ae3709f8685a443b1267820e1d48584b0cc Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 17 Jun 2026 11:04:34 +0200 Subject: [PATCH 12/40] chore: add tests for checking that all ng_ and nemo_gym_ commands print deprecation note Signed-off-by: Marta Stepniewska-Dziubinska --- tests/unit_tests/test_cli_legacy.py | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 tests/unit_tests/test_cli_legacy.py diff --git a/tests/unit_tests/test_cli_legacy.py b/tests/unit_tests/test_cli_legacy.py new file mode 100644 index 0000000000..cad72c0a61 --- /dev/null +++ b/tests/unit_tests/test_cli_legacy.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import tomllib + +import pytest +from pytest import MonkeyPatch + +import nemo_gym.cli.legacy as legacy +from nemo_gym import PARENT_DIR + + +def _legacy_scripts() -> list[tuple[str, str]]: + """All console scripts whose name is a legacy ng_*/nemo_gym_* alias.""" + with (PARENT_DIR / "pyproject.toml").open("rb") as f: + scripts = tomllib.load(f)["project"]["scripts"] + return [(name, target) for name, target in scripts.items() if name.startswith(("ng_", "nemo_gym_"))] + + +LEGACY_SCRIPTS = _legacy_scripts() + + +class TestLegacyDeprecation: + """Remove these tests once the legacy commands are removed.""" + + def test_legacy_scripts_were_discovered(self) -> None: + # Guard so the parametrized test below can't pass vacuously if discovery breaks. + assert len(LEGACY_SCRIPTS) > 1 + + @pytest.mark.parametrize("name, target", LEGACY_SCRIPTS) + def test_legacy_command_shows_deprecation(self, monkeypatch: MonkeyPatch, capsys, name: str, target: str) -> None: + # Every legacy alias must route through the shim, which prints a deprecation notice and keeps working. + assert target == "nemo_gym.cli.legacy:main", f"{name} should route through the legacy shim" + + # Stub the actual execution paths so nothing real runs. + monkeypatch.setattr(legacy, "gym_main", lambda: None) + monkeypatch.setattr(legacy, "dispatch", lambda *a, **k: None) + monkeypatch.setattr(sys, "argv", [name]) + + legacy.main() + + assert "deprecated" in capsys.readouterr().err From 62881709c72572f690975f1cc2aa4e5468b51bac Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 17 Jun 2026 11:58:09 +0200 Subject: [PATCH 13/40] feat: add --verbose flag Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 4 +++ nemo_gym/global_config.py | 13 +++++++++ tests/unit_tests/test_cli_main.py | 46 +++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 47e8d2efcd..9f24c2b4e4 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -312,6 +312,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: def _add_leaf(subparsers: argparse._SubParsersAction, name: str, command: Command) -> None: leaf = subparsers.add_parser(name, help=command.summary, description=command.summary) leaf.set_defaults(_command=command) + leaf.add_argument("-v", "--verbose", action="store_true", help="Set logging level to DEBUG.") for flag in command.flags: flag.register(leaf) @@ -361,6 +362,9 @@ def main() -> None: sys.exit(1) overrides = [token for flag in command.flags for token in flag.translate_to_hydra(args)] + overrides + # --verbose flows through the config (as +verbose=true) so it reaches spun-up servers, not just this process. + if getattr(args, "verbose", False): + overrides = ["+verbose=true", *overrides] if callable(command.target): command.target(args, overrides) else: diff --git a/nemo_gym/global_config.py b/nemo_gym/global_config.py index a614cecace..ae69c6696a 100644 --- a/nemo_gym/global_config.py +++ b/nemo_gym/global_config.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging from argparse import ArgumentParser from collections import defaultdict from copy import deepcopy @@ -68,6 +69,7 @@ COPY_KEY_NAME = "_copy" DELETE_KEY_KEY_NAME = "_delete_key" NEMO_GYM_LOG_DIR_KEY_NAME = "nemo_gym_log_dir" +VERBOSE_KEY_NAME = "verbose" NEMO_GYM_RESERVED_TOP_LEVEL_KEYS = [ CONFIG_PATHS_KEY_NAME, ENTRYPOINT_KEY_NAME, @@ -90,6 +92,7 @@ INHERIT_FROM_KEY_NAME, COPY_KEY_NAME, NEMO_GYM_LOG_DIR_KEY_NAME, + VERBOSE_KEY_NAME, ] # Data keys @@ -624,6 +627,7 @@ def get_global_config_dict( _GLOBAL_CONFIG_DICT = global_config_dict + _apply_verbosity(global_config_dict) return global_config_dict set_global_config_dict( @@ -631,9 +635,18 @@ def get_global_config_dict( global_config_dict_parser_cls=global_config_dict_parser_cls, ) + _apply_verbosity(_GLOBAL_CONFIG_DICT) return _GLOBAL_CONFIG_DICT +def _apply_verbosity(global_config_dict: DictConfig) -> None: + """Set logging to DEBUG when `verbose` is in the config. Runs in the CLI process and, because the + config dict is forwarded to every spun-up server, in each server process too.""" + if global_config_dict.get(VERBOSE_KEY_NAME): + logging.basicConfig(level=logging.DEBUG) + logging.getLogger().setLevel(logging.DEBUG) + + def set_global_config_dict( global_config_dict_parser_config: Optional[GlobalConfigDictParserConfig] = None, global_config_dict_parser_cls: Type[GlobalConfigDictParser] = GlobalConfigDictParser, diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 59f3a8f567..fe17298643 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -12,13 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging import sys import pytest from pytest import MonkeyPatch import nemo_gym.cli.main as cli_main +import nemo_gym.global_config as gc from nemo_gym.cli.main import main +from nemo_gym.global_config import NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME def _dispatch_for(monkeypatch: MonkeyPatch, argv: list[str]) -> tuple[str, list[str]]: @@ -462,3 +465,46 @@ def test_flags(self, monkeypatch: MonkeyPatch) -> None: "+entrypoint=resources_servers/gpqa", "+outdated=true", } + + +class TestVerboseFlag: + @pytest.mark.parametrize("flag", ["-v", "--verbose"]) + def test_verbose_injects_config_override(self, monkeypatch: MonkeyPatch, flag: str) -> None: + # --verbose flows through the config (so it reaches servers), not just the local logger. + _, overrides = _dispatch_for(monkeypatch, ["env", "status", flag]) + assert overrides == ["+verbose=true"] + + def test_no_verbose_no_override(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "status"]) + assert overrides == [] + + def test_verbose_prepended_before_other_overrides(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--verbose", "--agent", "a", "+x=1"]) + assert "+verbose=true" in overrides + assert "+agent_name=a" in overrides + assert "+x=1" in overrides + + def test_config_verbose_sets_debug_on_load(self, monkeypatch: MonkeyPatch) -> None: + # The server-side path: a config carrying `verbose` (forwarded via env var) raises the log level. + monkeypatch.setattr(gc, "_GLOBAL_CONFIG_DICT", None) + monkeypatch.setenv(NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, "verbose: true\nsome_server: {}\n") + root = logging.getLogger() + original = root.level + try: + root.setLevel(logging.WARNING) + gc.get_global_config_dict() + assert root.level == logging.DEBUG + finally: + root.setLevel(original) + + def test_config_without_verbose_keeps_level(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr(gc, "_GLOBAL_CONFIG_DICT", None) + monkeypatch.setenv(NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, "some_server: {}\n") + root = logging.getLogger() + original = root.level + try: + root.setLevel(logging.WARNING) + gc.get_global_config_dict() + assert root.level == logging.WARNING + finally: + root.setLevel(original) From 6a2cd5c554701bc9be737dc227f4ebb4a4728b7b Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 17 Jun 2026 13:57:40 +0200 Subject: [PATCH 14/40] feat: allow to select relevant config through passing server name Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 78 ++++++++++++- tests/unit_tests/test_cli_main.py | 183 +++++++++++++++++++++++++++++- 2 files changed, 257 insertions(+), 4 deletions(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 9f24c2b4e4..3e1f266b99 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -18,6 +18,8 @@ from collections.abc import Callable from dataclasses import dataclass, field +from nemo_gym import WORKING_DIR + VERSION_TARGET = "nemo_gym.cli.general:version" @@ -103,6 +105,65 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: ) +# Asset selector flag -> (parent dir, configs subdir, default config flavor). All accept `name` or `name/flavor`, +# resolving to `//[/].yaml`. A None default flavor falls back to the server name. +_ASSETS = { + "benchmark": ("benchmarks", "", "config"), + "resource-server": ("resources_servers", "configs", None), + "model-type": ("responses_api_models", "configs", None), +} + + +def _asset_config_path(flag: str, value: str) -> str: + """Map a named asset (`name` or `name/flavor`) to its config path, verifying the file exists under WORKING_DIR.""" + parent, subdir, default_flavor = _ASSETS[flag] + server_name, _, config_flavor = value.partition("/") + config_flavor = config_flavor or default_flavor or server_name + config_dir = f"{parent}/{server_name}/{subdir}".rstrip("/") + path = f"{config_dir}/{config_flavor}.yaml" + + if (WORKING_DIR / path).exists(): + return path + + if (WORKING_DIR / parent / server_name).exists(): + available = f"{config_dir}/" + else: + available = f"{parent}/" + raise ValueError( + f"`--{flag} {value}` was specified which implies config `{path}`, which does not exist. " + f"See available {flag} configs in `{available}`." + ) + + +def _asset_selector(flag: str) -> Flag: + """A `-- NAME` selector that resolves the named asset to a config and adds it to +config_paths.""" + dest = flag.replace("-", "_") + return Flag( + register=lambda p: p.add_argument(f"--{flag}", metavar="NAME", help=f"Load the named {flag} config."), + translate_to_hydra=lambda args: ( + [f"+config_paths=[{_asset_config_path(flag, getattr(args, dest))}]"] if getattr(args, dest) else [] + ), + ) + + +BENCHMARK = _asset_selector("benchmark") +RESOURCE_SERVER_CONFIG = _asset_selector("resource-server") +MODEL_TYPE = _asset_selector("model-type") + + +def _merge_config_paths(overrides: list[str]) -> list[str]: + """Coalesce all `+config_paths=[...]` tokens (from --config and asset selectors) into one (Hydra rejects dupes).""" + prefix = "+config_paths=[" + paths: list[str] = [] + rest: list[str] = [] + for token in overrides: + if token.startswith(prefix) and token.endswith("]"): + paths.extend(p for p in token[len(prefix) : -1].split(",") if p) + else: + rest.append(token) + return ([f"+config_paths=[{','.join(paths)}]"] if paths else []) + rest + + def _eval_run(args: argparse.Namespace, overrides: list[str]) -> None: target = "nemo_gym.cli.eval:collect_rollouts" if args.no_serve else "nemo_gym.cli.eval:e2e_rollout_collection" dispatch(target, overrides) @@ -214,6 +275,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: summary="Validate and collate the dataset.", flags=( CONFIG, + RESOURCE_SERVER_CONFIG, _value_flag("mode", "mode", "Data preparation mode.", choices=("train_preparation", "example_validation")), _value_flag("output-dir", "output_dirpath", "Output directory for the prepared data."), _bool_flag("download", "should_download", "Download source datasets before collating."), @@ -245,19 +307,22 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "env run": Command( target="nemo_gym.cli.env:run", summary="Start the servers.", - flags=(CONFIG, MODEL_NAME, MODEL_URL, MODEL_API_KEY), + flags=(CONFIG, RESOURCE_SERVER_CONFIG, MODEL_TYPE, MODEL_NAME, MODEL_URL, MODEL_API_KEY), ), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status."), "eval prepare": Command( target="nemo_gym.cli.eval:prepare_benchmark", summary="Prepare benchmark data and dump it to disk.", - flags=(CONFIG,), + flags=(CONFIG, BENCHMARK), ), "eval run": Command( target=_eval_run, summary="Collate data, start servers, and collect rollouts.", flags=( CONFIG, + BENCHMARK, + RESOURCE_SERVER_CONFIG, + MODEL_TYPE, Flag( register=lambda p: p.add_argument( "--no-serve", @@ -361,7 +426,14 @@ def main() -> None: args._parser.print_help() sys.exit(1) - overrides = [token for flag in command.flags for token in flag.translate_to_hydra(args)] + overrides + try: + translated = [token for flag in command.flags for token in flag.translate_to_hydra(args)] + except ValueError as exc: + sys.stderr.write(f"{getattr(args, '_parser', parser).prog}: error: {exc}\n") + sys.exit(2) + + # --config and the asset selectors all emit +config_paths; coalesce them into one token. + overrides = _merge_config_paths(translated + overrides) # --verbose flows through the config (as +verbose=true) so it reaches spun-up servers, not just this process. if getattr(args, "verbose", False): overrides = ["+verbose=true", *overrides] diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index fe17298643..5133b5caf3 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -38,6 +38,16 @@ def fake_dispatch(target: str, overrides: list[str]) -> None: return captured["target"], captured["overrides"] +def _split_overrides(overrides: list[str]) -> tuple[set[str], set[str]]: + """Split overrides into (config paths, other overrides) as sets, so tests never assert ordering.""" + prefix = "+config_paths=[" + config_tokens = [o for o in overrides if o.startswith(prefix) and o.endswith("]")] + assert len(config_tokens) <= 1 # --config and the asset selectors coalesce into a single token + paths = set(config_tokens[0][len(prefix) : -1].split(",")) if config_tokens else set() + others = {o for o in overrides if o not in config_tokens} + return paths, others + + # `gym ` -> the legacy ng_ function it dispatches to, for the config-accepting commands. CONFIG_COMMANDS = [ (["env", "run"], "nemo_gym.cli.env:run"), @@ -362,7 +372,11 @@ def test_render(self, monkeypatch: MonkeyPatch) -> None: monkeypatch, ["dataset", "render", "-i", "raw.jsonl", "--prompt-config", "p.yaml", "-o", "out.jsonl"] ) assert target == "nemo_gym.cli.dataset:materialize_prompts_cli" - assert overrides == ["+input_jsonl_fpath=raw.jsonl", "+prompt_config=p.yaml", "+output_jsonl_fpath=out.jsonl"] + assert set(overrides) == { + "+input_jsonl_fpath=raw.jsonl", + "+prompt_config=p.yaml", + "+output_jsonl_fpath=out.jsonl", + } def test_collate(self, monkeypatch: MonkeyPatch) -> None: target, overrides = _dispatch_for( @@ -508,3 +522,170 @@ def test_config_without_verbose_keeps_level(self, monkeypatch: MonkeyPatch) -> N assert root.level == logging.WARNING finally: root.setLevel(original) + + +class TestAssetSelectors: + """Named selectors (--benchmark, --resource-server, --model-type) that resolve a name to a default config path. + + Each example mirrors a real invocation from the docs/READMEs, so the sugar stays faithful to the documented + config paths it replaces. The legacy `+config_paths=[...]` form each one is derived from is cited inline. + """ + + @pytest.mark.parametrize( + "argv, expected_config", + [ + # benchmarks/gsm8k/README.md: ng_prepare_benchmark "+config_paths=[benchmarks/gsm8k/config.yaml]" + (["eval", "prepare", "--benchmark", "gsm8k"], "benchmarks/gsm8k/config.yaml"), + # benchmarks/aime25-x/README.md: ng_prepare_benchmark "+config_paths=[benchmarks/aime25-x/config.yaml]" + (["eval", "prepare", "--benchmark", "aime25-x"], "benchmarks/aime25-x/config.yaml"), + # README.md / quickstart.mdx: resources_servers/mcqa/configs/mcqa.yaml + (["env", "run", "--resource-server", "mcqa"], "resources_servers/mcqa/configs/mcqa.yaml"), + # model-server/vllm.mdx: resources_servers/example_multi_step/configs/example_multi_step.yaml + ( + ["env", "run", "--resource-server", "example_multi_step"], + "resources_servers/example_multi_step/configs/example_multi_step.yaml", + ), + # README.md / quickstart.mdx: responses_api_models/openai_model/configs/openai_model.yaml + ( + ["env", "run", "--model-type", "openai_model"], + "responses_api_models/openai_model/configs/openai_model.yaml", + ), + # model-server/vllm.mdx: responses_api_models/vllm_model/configs/vllm_model.yaml + (["env", "run", "--model-type", "vllm_model"], "responses_api_models/vllm_model/configs/vllm_model.yaml"), + ], + ) + def test_name_resolves_to_config_path(self, monkeypatch: MonkeyPatch, argv, expected_config) -> None: + _, overrides = _dispatch_for(monkeypatch, argv) + assert overrides == [f"+config_paths=[{expected_config}]"] + + def test_quickstart_resource_server_plus_model(self, monkeypatch: MonkeyPatch) -> None: + # README.md / quickstart.mdx: + # ng_run "+config_paths=[resources_servers/mcqa/configs/mcqa.yaml, + # responses_api_models/openai_model/configs/openai_model.yaml]" + target, overrides = _dispatch_for( + monkeypatch, ["env", "run", "--resource-server", "mcqa", "--model-type", "openai_model"] + ) + assert target == "nemo_gym.cli.env:run" + paths, others = _split_overrides(overrides) + assert paths == { + "resources_servers/mcqa/configs/mcqa.yaml", + "responses_api_models/openai_model/configs/openai_model.yaml", + } + assert others == set() + + def test_gpqa_benchmark_plus_model(self, monkeypatch: MonkeyPatch) -> None: + # benchmarks/gpqa/README.md: + # ng_run "+config_paths=[benchmarks/gpqa/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" + _, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--benchmark", "gpqa", "--model-type", "vllm_model"]) + paths, others = _split_overrides(overrides) + assert paths == { + "benchmarks/gpqa/config.yaml", + "responses_api_models/vllm_model/configs/vllm_model.yaml", + } + assert others == set() + + def test_cli_reference_e2e_rollout_example(self, monkeypatch: MonkeyPatch) -> None: + # fern .../reference/cli-commands.mdx ng_e2e_collect_rollouts example: + # config_paths="responses_api_models/openai_model/configs/openai_model.yaml, + # resources_servers/math_with_judge/configs/math_with_judge.yaml" + # ng_e2e_collect_rollouts "+config_paths=[$config_paths]" + # ++output_jsonl_fpath=results/test_e2e_rollout_collection/aime24.jsonl ++split=validation + target, overrides = _dispatch_for( + monkeypatch, + [ + "eval", + "run", + "--resource-server", + "math_with_judge", + "--model-type", + "openai_model", + "--output", + "results/test_e2e_rollout_collection/aime24.jsonl", + "--split", + "validation", + ], + ) + assert target == "nemo_gym.cli.eval:e2e_rollout_collection" + paths, others = _split_overrides(overrides) + assert paths == { + "resources_servers/math_with_judge/configs/math_with_judge.yaml", + "responses_api_models/openai_model/configs/openai_model.yaml", + } + assert others == { + "+output_jsonl_fpath=results/test_e2e_rollout_collection/aime24.jsonl", + "+split=validation", + } + + def test_cli_reference_prepare_data_example(self, monkeypatch: MonkeyPatch) -> None: + # fern .../reference/cli-commands.mdx ng_prepare_data example: + # config_paths includes resources_servers/example_multi_step/configs/example_multi_step.yaml + # ng_prepare_data "+config_paths=[...]" +output_dirpath=data/example_multi_step +mode=example_validation + target, overrides = _dispatch_for( + monkeypatch, + [ + "dataset", + "collate", + "--resource-server", + "example_multi_step", + "--mode", + "example_validation", + "--output-dir", + "data/example_multi_step", + ], + ) + assert target == "nemo_gym.cli.dataset:prepare_data" + paths, others = _split_overrides(overrides) + assert paths == {"resources_servers/example_multi_step/configs/example_multi_step.yaml"} + assert others == { + "+mode=example_validation", + "+output_dirpath=data/example_multi_step", + } + + def test_resource_server_flavor_syntax(self, monkeypatch: MonkeyPatch) -> None: + # `/` picks a named config inside the server's configs/ dir; math_with_judge ships several + # flavoured configs (see reference/faq.mdx, which pairs a math_with_judge dataset flavour for profiling). + _, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--resource-server", "math_with_judge/dapo17k"]) + assert overrides == ["+config_paths=[resources_servers/math_with_judge/configs/dapo17k.yaml]"] + + def test_benchmark_flavor_syntax(self, monkeypatch: MonkeyPatch) -> None: + # Benchmarks are flavoured too: flavor is a sibling `.yaml` (no configs/ dir), default `config`. + # e.g. benchmarks/finance_sec_search ships config_web_search.yaml alongside the default config.yaml. + _, overrides = _dispatch_for( + monkeypatch, ["eval", "prepare", "--benchmark", "finance_sec_search/config_web_search"] + ) + assert overrides == ["+config_paths=[benchmarks/finance_sec_search/config_web_search.yaml]"] + + def test_selectors_merge_into_single_config_paths(self, monkeypatch: MonkeyPatch) -> None: + # --config and multiple asset selectors all feed one +config_paths list (Hydra rejects duplicates). + # _split_overrides asserts they coalesce into a single token. + _, overrides = _dispatch_for( + monkeypatch, + ["eval", "run", "--config", "extra.yaml", "--resource-server", "mcqa", "--model-type", "openai_model"], + ) + paths, others = _split_overrides(overrides) + assert paths == { + "extra.yaml", + "resources_servers/mcqa/configs/mcqa.yaml", + "responses_api_models/openai_model/configs/openai_model.yaml", + } + assert others == set() + + def test_unknown_benchmark_errors_with_available_hint(self, monkeypatch: MonkeyPatch, capsys) -> None: + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", "eval", "prepare", "--benchmark", "does_not_exist"]) + with pytest.raises(SystemExit): + main() + err = capsys.readouterr().err + assert "benchmarks/does_not_exist/config.yaml" in err + assert "does not exist" in err + assert "benchmarks/" in err + + def test_unknown_flavor_error_points_at_configs_dir(self, monkeypatch: MonkeyPatch, capsys) -> None: + # For a known server with an unknown flavor, the hint should point at that server's configs/ dir. + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", "env", "run", "--resource-server", "mcqa/nope"]) + with pytest.raises(SystemExit): + main() + err = capsys.readouterr().err + assert "resources_servers/mcqa/configs/nope.yaml" in err + assert "resources_servers/mcqa/configs/" in err From 16431a64d94f6cbf8380b6b7eda64de69c4581d0 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 17 Jun 2026 14:39:57 +0200 Subject: [PATCH 15/40] feat: add --json flag for machine-readable output (+ move diagnostics to stderr) Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/env.py | 6 +++++ nemo_gym/cli/eval.py | 10 ++++++++ nemo_gym/cli/general.py | 10 +++----- nemo_gym/cli/main.py | 19 +++++++++++--- nemo_gym/global_config.py | 9 +++++++ nemo_gym/server_status.py | 12 ++++++--- tests/unit_tests/test_benchmarks.py | 23 +++++++++++++++++ tests/unit_tests/test_cli_main.py | 35 ++++++++++++++++++++++++++ tests/unit_tests/test_server_status.py | 13 ++++++---- 9 files changed, 119 insertions(+), 18 deletions(-) diff --git a/nemo_gym/cli/env.py b/nemo_gym/cli/env.py index 69c422454b..2c57f8aa80 100644 --- a/nemo_gym/cli/env.py +++ b/nemo_gym/cli/env.py @@ -41,6 +41,7 @@ from nemo_gym.config_types import BaseNeMoGymCLIConfig from nemo_gym.global_config import ( DRY_RUN_KEY_NAME, + JSON_OUTPUT_KEY_NAME, NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, NEMO_GYM_CONFIG_PATH_ENV_VAR_NAME, NEMO_GYM_RESERVED_TOP_LEVEL_KEYS, @@ -805,6 +806,11 @@ def status(): # pragma: no cover status_cmd = StatusCommand() servers = status_cmd.discover_servers() + + if global_config_dict.get(JSON_OUTPUT_KEY_NAME, False): + print(json.dumps([server.model_dump(mode="json") for server in servers])) + return + status_cmd.display_status(servers) diff --git a/nemo_gym/cli/eval.py b/nemo_gym/cli/eval.py index 54f9b25a84..473083a358 100644 --- a/nemo_gym/cli/eval.py +++ b/nemo_gym/cli/eval.py @@ -15,6 +15,7 @@ import asyncio import importlib +import json from copy import deepcopy from glob import glob from multiprocessing import Pool @@ -32,6 +33,7 @@ from nemo_gym.cli.env import RunHelper from nemo_gym.config_types import BaseNeMoGymCLIConfig, BenchmarkDatasetConfig from nemo_gym.global_config import ( + JSON_OUTPUT_KEY_NAME, ROLLOUT_INDEX_KEY_NAME, TASK_INDEX_KEY_NAME, GlobalConfigDictParserConfig, @@ -66,6 +68,14 @@ def list_benchmarks() -> None: benchmarks = _load_benchmarks_from_config_paths(config_paths) + if global_config_dict.get(JSON_OUTPUT_KEY_NAME, False): + payload = [ + {"name": name, "agent_name": bench.agent_name, "num_repeats": bench.num_repeats} + for name, bench in benchmarks.items() + ] + print(json.dumps(payload)) + return + if not benchmarks: rich.print("[yellow]No benchmarks found.[/yellow]") rich.print(f"Expected benchmarks directory: {BENCHMARKS_DIR}") diff --git a/nemo_gym/cli/general.py b/nemo_gym/cli/general.py index b085e97b3f..3c577e8e88 100644 --- a/nemo_gym/cli/general.py +++ b/nemo_gym/cli/general.py @@ -27,7 +27,7 @@ from nemo_gym import PARENT_DIR, __version__ from nemo_gym.config_types import BaseNeMoGymCLIConfig -from nemo_gym.global_config import get_global_config_dict +from nemo_gym.global_config import JSON_OUTPUT_KEY_NAME, get_global_config_dict def display_help_legacy(): @@ -78,9 +78,8 @@ class VersionConfig(BaseNeMoGymCLIConfig): def version(): # pragma: no cover """Display gym version and system information.""" global_config_dict = get_global_config_dict() - config = VersionConfig.model_validate(global_config_dict) - - json_output = config.json_format + # Just here for help. + VersionConfig.model_validate(global_config_dict) version_info = { "nemo_gym": __version__, @@ -111,8 +110,7 @@ def version(): # pragma: no cover mem = psutil.virtual_memory() version_info["system"]["memory_gb"] = round(mem.total / (1024**3), 2) - # Output - if json_output: + if global_config_dict.get(JSON_OUTPUT_KEY_NAME, False): print(json.dumps(version_info)) else: output = f"""\ diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 3e1f266b99..62fac8aace 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -104,6 +104,10 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: ), ) +# Shared flag: emit machine-readable JSON instead of human output. Reused by reporting commands (version, list, +# env status). The reserved `json` config key is read centrally via nemo_gym.cli.output.emit. +JSON = _bool_flag("json", "json", "Output as JSON for programmatic use.") + # Asset selector flag -> (parent dir, configs subdir, default config flavor). All accept `name` or `name/flavor`, # resolving to `//[/].yaml`. A None default flavor falls back to the server name. @@ -203,7 +207,9 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: } COMMANDS = { - "list benchmarks": Command(target="nemo_gym.cli.eval:list_benchmarks", summary="List available benchmarks."), + "list benchmarks": Command( + target="nemo_gym.cli.eval:list_benchmarks", summary="List available benchmarks.", flags=(JSON,) + ), "dataset upload": Command( target=_dataset_upload, summary="Upload a prepared dataset to HF (default) or GitLab.", @@ -297,6 +303,12 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: flags=( RESOURCE_SERVER, _bool_flag("outdated", "outdated", "List only outdated packages."), + Flag( + register=lambda p: p.add_argument( + "--json", action="store_true", help="Output the package list as JSON." + ), + translate_to_hydra=lambda args: ["+format=json"] if args.json else [], + ), ), ), "env test": Command( @@ -309,7 +321,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: summary="Start the servers.", flags=(CONFIG, RESOURCE_SERVER_CONFIG, MODEL_TYPE, MODEL_NAME, MODEL_URL, MODEL_API_KEY), ), - "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status."), + "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status.", flags=(JSON,)), "eval prepare": Command( target="nemo_gym.cli.eval:prepare_benchmark", summary="Prepare benchmark data and dump it to disk.", @@ -385,6 +397,7 @@ def _add_leaf(subparsers: argparse._SubParsersAction, name: str, command: Comman def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="gym", add_help=True) parser.add_argument("--version", action="store_true", help="Show the NeMo Gym version and exit.") + parser.add_argument("--json", action="store_true", help="With --version, output as JSON.") parser.set_defaults(_parser=parser) subparsers = parser.add_subparsers() @@ -418,7 +431,7 @@ def main() -> None: getattr(args, "_parser", parser).error(f"unrecognized arguments: {' '.join(unknown_flags)}") if args.version: - dispatch(VERSION_TARGET, overrides) + dispatch(VERSION_TARGET, ["+json=true", *overrides] if args.json else overrides) return command = getattr(args, "_command", None) diff --git a/nemo_gym/global_config.py b/nemo_gym/global_config.py index ae69c6696a..006808a48f 100644 --- a/nemo_gym/global_config.py +++ b/nemo_gym/global_config.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging +import sys from argparse import ArgumentParser from collections import defaultdict from copy import deepcopy @@ -70,6 +71,7 @@ DELETE_KEY_KEY_NAME = "_delete_key" NEMO_GYM_LOG_DIR_KEY_NAME = "nemo_gym_log_dir" VERBOSE_KEY_NAME = "verbose" +JSON_OUTPUT_KEY_NAME = "json" NEMO_GYM_RESERVED_TOP_LEVEL_KEYS = [ CONFIG_PATHS_KEY_NAME, ENTRYPOINT_KEY_NAME, @@ -93,6 +95,7 @@ COPY_KEY_NAME, NEMO_GYM_LOG_DIR_KEY_NAME, VERBOSE_KEY_NAME, + JSON_OUTPUT_KEY_NAME, ] # Data keys @@ -186,6 +189,12 @@ def inner_hydra_wrapper(cfg: DictConfig) -> DictConfig: inner_hydra_wrapper() + # Hydra installs a console log handler on stdout; move it to stderr so command stdout stays machine-readable + # (e.g. `gym ... --json`). Diagnostics belong on stderr; only the requested data goes to stdout. + for handler in logging.getLogger().handlers: + if isinstance(handler, logging.StreamHandler) and getattr(handler, "stream", None) is sys.stdout: + handler.setStream(sys.stderr) + global_config_dict: DictConfig = config_list[0] return global_config_dict diff --git a/nemo_gym/server_status.py b/nemo_gym/server_status.py index cfa65eb067..b4166cb94d 100644 --- a/nemo_gym/server_status.py +++ b/nemo_gym/server_status.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging from time import time from typing import List @@ -21,6 +22,9 @@ from nemo_gym.server_utils import ServerClient, ServerInstanceDisplayConfig, ServerStatus +logger = logging.getLogger(__name__) + + class StatusCommand: """Main class to check server status""" @@ -73,10 +77,10 @@ def discover_servers(self) -> List[ServerInstanceDisplayConfig]: return servers except (requests.RequestException, ConnectionError) as e: - print(f""" -Could not connect to head server: {e} -Is the head server running? Start it with: `ng_run` - """) + logger.warning( + "Could not connect to head server: %s. Is the head server running? Start it with: `gym env run`", + e, + ) return [] def display_status(self, servers: List[ServerInstanceDisplayConfig]) -> None: diff --git a/tests/unit_tests/test_benchmarks.py b/tests/unit_tests/test_benchmarks.py index 653788858f..a104a43490 100644 --- a/tests/unit_tests/test_benchmarks.py +++ b/tests/unit_tests/test_benchmarks.py @@ -41,6 +41,29 @@ def test_no_benchmarks(self, capsys) -> None: list_benchmarks() assert "No benchmarks found" in capsys.readouterr().out + def test_json_output(self, capsys) -> None: + import json + + bench = MagicMock(agent_name="my_agent", num_repeats=4) + with ( + patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config({"json": True})), + patch("nemo_gym.cli.eval._load_benchmarks_from_config_paths", return_value={"my_bench": bench}), + ): + list_benchmarks() + assert json.loads(capsys.readouterr().out) == [ + {"name": "my_bench", "agent_name": "my_agent", "num_repeats": 4} + ] + + def test_json_output_empty(self, capsys) -> None: + import json + + with ( + patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config({"json": True})), + patch("nemo_gym.cli.eval._load_benchmarks_from_config_paths", return_value={}), + ): + list_benchmarks() + assert json.loads(capsys.readouterr().out) == [] + class TestPrepareBenchmark: def _make_bench_dir(self, tmp_path: Path, name: str = "fake_bench") -> tuple[Path, Path]: diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 5133b5caf3..98000c3508 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -481,6 +481,41 @@ def test_flags(self, monkeypatch: MonkeyPatch) -> None: } +class TestJsonFlag: + @pytest.mark.parametrize( + "argv, expected_target", + [ + (["list", "benchmarks", "--json"], "nemo_gym.cli.eval:list_benchmarks"), + (["env", "status", "--json"], "nemo_gym.cli.env:status"), + ], + ) + def test_json_becomes_config_override(self, monkeypatch: MonkeyPatch, argv, expected_target) -> None: + # Reporting commands surface --json as the reserved `json` config key, read centrally by cli.output.emit. + target, overrides = _dispatch_for(monkeypatch, argv) + assert target == expected_target + assert overrides == ["+json=true"] + + def test_no_json_no_override(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["list", "benchmarks"]) + assert overrides == [] + + def test_version_json_dispatches_with_override(self, monkeypatch: MonkeyPatch) -> None: + # `gym --version --json` is the top-level path; it still forwards +json=true to the version command. + target, overrides = _dispatch_for(monkeypatch, ["--version", "--json"]) + assert target == "nemo_gym.cli.general:version" + assert overrides == ["+json=true"] + + def test_env_packages_json_maps_to_uv_format(self, monkeypatch: MonkeyPatch) -> None: + # env packages delegates to `uv pip list`, so --json maps onto its own --format=json rather than +json=true. + target, overrides = _dispatch_for(monkeypatch, ["env", "packages", "--resource-server", "mcqa", "--json"]) + assert target == "nemo_gym.cli.env:pip_list" + assert set(overrides) == {"+entrypoint=resources_servers/mcqa", "+format=json"} + + def test_env_packages_without_json(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "packages", "--resource-server", "mcqa"]) + assert overrides == ["+entrypoint=resources_servers/mcqa"] + + class TestVerboseFlag: @pytest.mark.parametrize("flag", ["-v", "--verbose"]) def test_verbose_injects_config_override(self, monkeypatch: MonkeyPatch, flag: str) -> None: diff --git a/tests/unit_tests/test_server_status.py b/tests/unit_tests/test_server_status.py index e7fdd26dd5..3157afad8a 100644 --- a/tests/unit_tests/test_server_status.py +++ b/tests/unit_tests/test_server_status.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging from io import StringIO from unittest.mock import MagicMock @@ -208,7 +209,7 @@ def test_discover_servers(self, monkeypatch: MonkeyPatch) -> None: first_call = mock_get.call_args_list[0] assert first_call[0][0] == "http://127.0.0.1:11000/server_instances" - def test_discover_servers_head_server_down(self, monkeypatch: MonkeyPatch, capsys) -> None: + def test_discover_servers_head_server_down(self, monkeypatch: MonkeyPatch, capsys, caplog) -> None: mock_head_config = MagicMock() mock_head_config.host = "127.0.0.1" mock_head_config.port = 11000 @@ -219,9 +220,11 @@ def test_discover_servers_head_server_down(self, monkeypatch: MonkeyPatch, capsy monkeypatch.setattr(requests, "get", mock_get) cmd = StatusCommand() - servers = cmd.discover_servers() + with caplog.at_level(logging.WARNING): + servers = cmd.discover_servers() assert len(servers) == 0 - captured = capsys.readouterr() - assert "Could not connect to head server" in captured.out - assert "ng_run" in captured.out + # The warning goes through logging (stderr), keeping stdout machine-readable for `--json`. + assert capsys.readouterr().out == "" + assert "Could not connect to head server" in caplog.text + assert "gym env run" in caplog.text From 02967e05c5b9cf942b561d22da443e7dd9e0d6a4 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 17 Jun 2026 15:03:20 +0200 Subject: [PATCH 16/40] feat: add 'did you mean?' hints for typos Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 43 +++++++++++++++++++++++++---- tests/unit_tests/test_cli_main.py | 46 +++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 62fac8aace..5995e61048 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -13,9 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import argparse +import difflib import importlib +import re import sys -from collections.abc import Callable +from collections.abc import Callable, Iterable from dataclasses import dataclass, field from nemo_gym import WORKING_DIR @@ -24,6 +26,27 @@ VERSION_TARGET = "nemo_gym.cli.general:version" +def _did_you_mean(value: str, candidates: Iterable[str]) -> str: + """A ` Did you mean \\`X\\`?` fragment for the closest candidate to `value`, or `""` if none is close enough.""" + matches = difflib.get_close_matches(value, list(candidates), n=1) + return f" Did you mean `{matches[0]}`?" if matches else "" + + +class _GymArgumentParser(argparse.ArgumentParser): + """ArgumentParser that appends a difflib "did you mean?" hint to invalid-choice errors. + + Covers mistyped commands/groups and bad --flag choices (e.g. --storage), since argparse validates all of them + as choices against the registry baked into the parser. + """ + + def error(self, message: str) -> None: + match = re.search(r"invalid choice: '([^']+)' \(choose from (.+)\)", message) + if match: + typo, choices = match.group(1), re.findall(r"'([^']+)'", match.group(2)) + message += _did_you_mean(typo, choices) + super().error(message) + + @dataclass(frozen=True) class Flag: # Register this flag's argument(s) on a command's subparser. @@ -129,12 +152,17 @@ def _asset_config_path(flag: str, value: str) -> str: if (WORKING_DIR / path).exists(): return path + # Suggest the closest real name: a config flavor when the server exists, otherwise a server name. if (WORKING_DIR / parent / server_name).exists(): available = f"{config_dir}/" + typo = config_flavor + candidates = [p.stem for p in (WORKING_DIR / config_dir).glob("*.yaml")] else: available = f"{parent}/" + typo = server_name + candidates = [child.name for child in (WORKING_DIR / parent).iterdir() if child.is_dir()] raise ValueError( - f"`--{flag} {value}` was specified which implies config `{path}`, which does not exist. " + f"`--{flag} {value}` was specified which implies config `{path}`, which does not exist.{_did_you_mean(typo, candidates)} " f"See available {flag} configs in `{available}`." ) @@ -388,14 +416,16 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: def _add_leaf(subparsers: argparse._SubParsersAction, name: str, command: Command) -> None: leaf = subparsers.add_parser(name, help=command.summary, description=command.summary) - leaf.set_defaults(_command=command) + # `_parser=leaf` so error reporting (and flag "did you mean?" hints) uses this command's own options/prog. + leaf.set_defaults(_command=command, _parser=leaf) leaf.add_argument("-v", "--verbose", action="store_true", help="Set logging level to DEBUG.") for flag in command.flags: flag.register(leaf) def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="gym", add_help=True) + # _GymArgumentParser propagates to every subparser (argparse defaults parser_class to type(self)). + parser = _GymArgumentParser(prog="gym", add_help=True) parser.add_argument("--version", action="store_true", help="Show the NeMo Gym version and exit.") parser.add_argument("--json", action="store_true", help="With --version, output as JSON.") parser.set_defaults(_parser=parser) @@ -428,7 +458,10 @@ def main() -> None: # Hydra overrides never start with "-" so we treat them as unknown flags. unknown_flags = [token for token in overrides if token.startswith("-")] if unknown_flags: - getattr(args, "_parser", parser).error(f"unrecognized arguments: {' '.join(unknown_flags)}") + error_parser = getattr(args, "_parser", parser) + known_options = [opt for action in error_parser._actions for opt in action.option_strings] + hints = "".join(_did_you_mean(flag.split("=", 1)[0], known_options) for flag in unknown_flags) + error_parser.error(f"unrecognized arguments: {' '.join(unknown_flags)}{hints}") if args.version: dispatch(VERSION_TARGET, ["+json=true", *overrides] if args.json else overrides) diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 98000c3508..a825f3aca6 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -724,3 +724,49 @@ def test_unknown_flavor_error_points_at_configs_dir(self, monkeypatch: MonkeyPat err = capsys.readouterr().err assert "resources_servers/mcqa/configs/nope.yaml" in err assert "resources_servers/mcqa/configs/" in err + + +class TestDidYouMean: + """difflib-backed "did you mean?" hints for mistyped commands, flags, and component names (proposal UX 4).""" + + def test_helper_suggests_close_match(self) -> None: + assert cli_main._did_you_mean("evl", ["list", "eval", "env"]) == " Did you mean `eval`?" + + def test_helper_silent_when_nothing_close(self) -> None: + assert cli_main._did_you_mean("zzzzzz", ["list", "eval", "env"]) == "" + + def _run_expecting_exit(self, monkeypatch: MonkeyPatch, capsys, argv: list[str]) -> str: + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", *argv]) + with pytest.raises(SystemExit): + main() + return capsys.readouterr().err + + def test_mistyped_group(self, monkeypatch: MonkeyPatch, capsys) -> None: + err = self._run_expecting_exit(monkeypatch, capsys, ["evl"]) + assert "invalid choice: 'evl'" in err + assert "Did you mean `eval`?" in err + + def test_mistyped_action(self, monkeypatch: MonkeyPatch, capsys) -> None: + err = self._run_expecting_exit(monkeypatch, capsys, ["eval", "rnu"]) + assert "Did you mean `run`?" in err + + def test_mistyped_flag_choice(self, monkeypatch: MonkeyPatch, capsys) -> None: + # --storage validates choices, so the parser-level hint kicks in. + err = self._run_expecting_exit(monkeypatch, capsys, ["dataset", "upload", "--storage", "gitlb"]) + assert "Did you mean `gitlab`?" in err + + def test_misspelled_flag(self, monkeypatch: MonkeyPatch, capsys) -> None: + err = self._run_expecting_exit(monkeypatch, capsys, ["eval", "run", "--benchmrk", "aalcr"]) + assert "unrecognized arguments: --benchmrk" in err + assert "Did you mean `--benchmark`?" in err + + def test_misspelled_component_name(self, monkeypatch: MonkeyPatch, capsys) -> None: + err = self._run_expecting_exit(monkeypatch, capsys, ["eval", "prepare", "--benchmark", "aalcrr"]) + assert "Did you mean `aalcr`?" in err + + def test_misspelled_component_flavor(self, monkeypatch: MonkeyPatch, capsys) -> None: + err = self._run_expecting_exit( + monkeypatch, capsys, ["eval", "run", "--resource-server", "math_with_judge/dapo17"] + ) + assert "Did you mean `dapo17k`?" in err From 81fca19fcb3af1468341f4fc2c9f8c651a86e8ac Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Mon, 22 Jun 2026 09:17:21 +0200 Subject: [PATCH 17/40] feat: add --search-dir for loading user configs from custom location Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 78 ++++++++++++++++++++++------- tests/unit_tests/test_cli_main.py | 82 ++++++++++++++++++++++++++----- 2 files changed, 130 insertions(+), 30 deletions(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 5995e61048..e525166d94 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -19,6 +19,7 @@ import sys from collections.abc import Callable, Iterable from dataclasses import dataclass, field +from pathlib import Path from nemo_gym import WORKING_DIR @@ -141,29 +142,55 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: } -def _asset_config_path(flag: str, value: str) -> str: - """Map a named asset (`name` or `name/flavor`) to its config path, verifying the file exists under WORKING_DIR.""" +def _asset_config_path(flag: str, value: str, search_dirs: tuple[str, ...] = ()) -> str: + """Map a named asset (`name` or `name/flavor`) to its config path. + + Searches WORKING_DIR (built-ins) first, then any user-registered --search-dir roots. + """ parent, subdir, default_flavor = _ASSETS[flag] server_name, _, config_flavor = value.partition("/") config_flavor = config_flavor or default_flavor or server_name config_dir = f"{parent}/{server_name}/{subdir}".rstrip("/") path = f"{config_dir}/{config_flavor}.yaml" - if (WORKING_DIR / path).exists(): - return path - - # Suggest the closest real name: a config flavor when the server exists, otherwise a server name. - if (WORKING_DIR / parent / server_name).exists(): - available = f"{config_dir}/" - typo = config_flavor - candidates = [p.stem for p in (WORKING_DIR / config_dir).glob("*.yaml")] - else: - available = f"{parent}/" + # Match in WORKING_DIR (built-ins) and every --search-dir root; dedupe roots that resolve to the same file. + roots = [WORKING_DIR, *(Path(d) for d in search_dirs)] + matches: list[Path] = [] + + for root in roots: + candidate = root / path + if candidate.exists(): + matches.append(candidate.resolve()) + + if len(matches) > 1: + matches_str = ", ".join(f"`{m}`" for m in matches) + raise ValueError( + f"`--{flag} {value}` is ambiguous: it matches multiple configs ({matches_str}). " + f"Pass the intended config directly with `--config ` instead." + ) + if matches: + return str(matches[0]) + + # No match: suggest the closest real name across all roots (a config flavor when the server exists, else a + # server name) and report the full paths that were searched. + available = ", ".join(set(f"`{(root / config_dir).resolve()}`" for root in roots if (root / config_dir).is_dir())) + typo = config_flavor + candidates = [p.stem for root in roots for p in (root / config_dir).glob("*.yaml")] + + if len(candidates) == 0: + available = ", ".join(set(f"`{(root / parent).resolve()}`" for root in roots if (root / parent).is_dir())) typo = server_name - candidates = [child.name for child in (WORKING_DIR / parent).iterdir() if child.is_dir()] + candidates = [ + child.name + for root in roots + if (root / parent).is_dir() + for child in (root / parent).iterdir() + if child.is_dir() + ] + raise ValueError( f"`--{flag} {value}` was specified which implies config `{path}`, which does not exist.{_did_you_mean(typo, candidates)} " - f"See available {flag} configs in `{available}`." + f"See available {flag} configs in {available}." ) @@ -173,7 +200,11 @@ def _asset_selector(flag: str) -> Flag: return Flag( register=lambda p: p.add_argument(f"--{flag}", metavar="NAME", help=f"Load the named {flag} config."), translate_to_hydra=lambda args: ( - [f"+config_paths=[{_asset_config_path(flag, getattr(args, dest))}]"] if getattr(args, dest) else [] + [ + f"+config_paths=[{_asset_config_path(flag, getattr(args, dest), tuple(getattr(args, 'search_dir', None) or ()))}]" + ] + if getattr(args, dest) + else [] ), ) @@ -182,6 +213,17 @@ def _asset_selector(flag: str) -> Flag: RESOURCE_SERVER_CONFIG = _asset_selector("resource-server") MODEL_TYPE = _asset_selector("model-type") +# Shared flag: register extra root dirs to search for named components. Consumed by the asset selectors above +# (not emitted as a Hydra override). Reused by every command that accepts a -- NAME selector. +SEARCH_DIR = Flag( + register=lambda p: p.add_argument( + "--search-dir", + action="append", + metavar="DIR", + help="Extra root directory to search for named components; repeatable.", + ), +) + def _merge_config_paths(overrides: list[str]) -> list[str]: """Coalesce all `+config_paths=[...]` tokens (from --config and asset selectors) into one (Hydra rejects dupes).""" @@ -310,6 +352,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: flags=( CONFIG, RESOURCE_SERVER_CONFIG, + SEARCH_DIR, _value_flag("mode", "mode", "Data preparation mode.", choices=("train_preparation", "example_validation")), _value_flag("output-dir", "output_dirpath", "Output directory for the prepared data."), _bool_flag("download", "should_download", "Download source datasets before collating."), @@ -347,13 +390,13 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "env run": Command( target="nemo_gym.cli.env:run", summary="Start the servers.", - flags=(CONFIG, RESOURCE_SERVER_CONFIG, MODEL_TYPE, MODEL_NAME, MODEL_URL, MODEL_API_KEY), + flags=(CONFIG, RESOURCE_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, MODEL_NAME, MODEL_URL, MODEL_API_KEY), ), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status.", flags=(JSON,)), "eval prepare": Command( target="nemo_gym.cli.eval:prepare_benchmark", summary="Prepare benchmark data and dump it to disk.", - flags=(CONFIG, BENCHMARK), + flags=(CONFIG, BENCHMARK, SEARCH_DIR), ), "eval run": Command( target=_eval_run, @@ -363,6 +406,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: BENCHMARK, RESOURCE_SERVER_CONFIG, MODEL_TYPE, + SEARCH_DIR, Flag( register=lambda p: p.add_argument( "--no-serve", diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index a825f3aca6..0a8c59a1ef 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -20,6 +20,7 @@ import nemo_gym.cli.main as cli_main import nemo_gym.global_config as gc +from nemo_gym import WORKING_DIR from nemo_gym.cli.main import main from nemo_gym.global_config import NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME @@ -591,7 +592,7 @@ class TestAssetSelectors: ) def test_name_resolves_to_config_path(self, monkeypatch: MonkeyPatch, argv, expected_config) -> None: _, overrides = _dispatch_for(monkeypatch, argv) - assert overrides == [f"+config_paths=[{expected_config}]"] + assert overrides == [f"+config_paths=[{WORKING_DIR / (expected_config)}]"] def test_quickstart_resource_server_plus_model(self, monkeypatch: MonkeyPatch) -> None: # README.md / quickstart.mdx: @@ -603,8 +604,8 @@ def test_quickstart_resource_server_plus_model(self, monkeypatch: MonkeyPatch) - assert target == "nemo_gym.cli.env:run" paths, others = _split_overrides(overrides) assert paths == { - "resources_servers/mcqa/configs/mcqa.yaml", - "responses_api_models/openai_model/configs/openai_model.yaml", + str(WORKING_DIR / "resources_servers/mcqa/configs/mcqa.yaml"), + str(WORKING_DIR / "responses_api_models/openai_model/configs/openai_model.yaml"), } assert others == set() @@ -614,8 +615,8 @@ def test_gpqa_benchmark_plus_model(self, monkeypatch: MonkeyPatch) -> None: _, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--benchmark", "gpqa", "--model-type", "vllm_model"]) paths, others = _split_overrides(overrides) assert paths == { - "benchmarks/gpqa/config.yaml", - "responses_api_models/vllm_model/configs/vllm_model.yaml", + str(WORKING_DIR / "benchmarks/gpqa/config.yaml"), + str(WORKING_DIR / "responses_api_models/vllm_model/configs/vllm_model.yaml"), } assert others == set() @@ -643,8 +644,8 @@ def test_cli_reference_e2e_rollout_example(self, monkeypatch: MonkeyPatch) -> No assert target == "nemo_gym.cli.eval:e2e_rollout_collection" paths, others = _split_overrides(overrides) assert paths == { - "resources_servers/math_with_judge/configs/math_with_judge.yaml", - "responses_api_models/openai_model/configs/openai_model.yaml", + str(WORKING_DIR / "resources_servers/math_with_judge/configs/math_with_judge.yaml"), + str(WORKING_DIR / "responses_api_models/openai_model/configs/openai_model.yaml"), } assert others == { "+output_jsonl_fpath=results/test_e2e_rollout_collection/aime24.jsonl", @@ -670,7 +671,7 @@ def test_cli_reference_prepare_data_example(self, monkeypatch: MonkeyPatch) -> N ) assert target == "nemo_gym.cli.dataset:prepare_data" paths, others = _split_overrides(overrides) - assert paths == {"resources_servers/example_multi_step/configs/example_multi_step.yaml"} + assert paths == {str(WORKING_DIR / "resources_servers/example_multi_step/configs/example_multi_step.yaml")} assert others == { "+mode=example_validation", "+output_dirpath=data/example_multi_step", @@ -680,7 +681,9 @@ def test_resource_server_flavor_syntax(self, monkeypatch: MonkeyPatch) -> None: # `/` picks a named config inside the server's configs/ dir; math_with_judge ships several # flavoured configs (see reference/faq.mdx, which pairs a math_with_judge dataset flavour for profiling). _, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--resource-server", "math_with_judge/dapo17k"]) - assert overrides == ["+config_paths=[resources_servers/math_with_judge/configs/dapo17k.yaml]"] + assert overrides == [ + f"+config_paths=[{WORKING_DIR / 'resources_servers/math_with_judge/configs/dapo17k.yaml'}]" + ] def test_benchmark_flavor_syntax(self, monkeypatch: MonkeyPatch) -> None: # Benchmarks are flavoured too: flavor is a sibling `.yaml` (no configs/ dir), default `config`. @@ -688,7 +691,7 @@ def test_benchmark_flavor_syntax(self, monkeypatch: MonkeyPatch) -> None: _, overrides = _dispatch_for( monkeypatch, ["eval", "prepare", "--benchmark", "finance_sec_search/config_web_search"] ) - assert overrides == ["+config_paths=[benchmarks/finance_sec_search/config_web_search.yaml]"] + assert overrides == [f"+config_paths=[{WORKING_DIR / 'benchmarks/finance_sec_search/config_web_search.yaml'}]"] def test_selectors_merge_into_single_config_paths(self, monkeypatch: MonkeyPatch) -> None: # --config and multiple asset selectors all feed one +config_paths list (Hydra rejects duplicates). @@ -699,9 +702,9 @@ def test_selectors_merge_into_single_config_paths(self, monkeypatch: MonkeyPatch ) paths, others = _split_overrides(overrides) assert paths == { - "extra.yaml", - "resources_servers/mcqa/configs/mcqa.yaml", - "responses_api_models/openai_model/configs/openai_model.yaml", + "extra.yaml", # raw --config value passes through verbatim; only name selectors get rooted + str(WORKING_DIR / "resources_servers/mcqa/configs/mcqa.yaml"), + str(WORKING_DIR / "responses_api_models/openai_model/configs/openai_model.yaml"), } assert others == set() @@ -770,3 +773,56 @@ def test_misspelled_component_flavor(self, monkeypatch: MonkeyPatch, capsys) -> monkeypatch, capsys, ["eval", "run", "--resource-server", "math_with_judge/dapo17"] ) assert "Did you mean `dapo17k`?" in err + + +class TestSearchDir: + """--search-dir registers extra roots that the name->config selectors also search (REQ 5).""" + + def _make_user_benchmark(self, tmp_path, name: str = "mybench") -> None: + bench_dir = tmp_path / "benchmarks" / name + bench_dir.mkdir(parents=True) + (bench_dir / "config.yaml").write_text("{}\n") + + def test_resolves_component_from_user_dir(self, monkeypatch: MonkeyPatch, tmp_path) -> None: + self._make_user_benchmark(tmp_path) + _, overrides = _dispatch_for( + monkeypatch, ["eval", "prepare", "--benchmark", "mybench", "--search-dir", str(tmp_path)] + ) + # User-dir matches are returned rooted so Hydra can resolve them. + assert overrides == [f"+config_paths=[{tmp_path / 'benchmarks' / 'mybench' / 'config.yaml'}]"] + + def test_builtin_resolves_when_search_dir_lacks_it(self, monkeypatch: MonkeyPatch, tmp_path) -> None: + # A built-in still resolves under WORKING_DIR when a --search-dir is provided that does not shadow it. + _, overrides = _dispatch_for( + monkeypatch, ["eval", "prepare", "--benchmark", "gsm8k", "--search-dir", str(tmp_path)] + ) + assert overrides == [f"+config_paths=[{WORKING_DIR / 'benchmarks/gsm8k/config.yaml'}]"] + + def test_ambiguous_match_errors(self, monkeypatch: MonkeyPatch, tmp_path, capsys) -> None: + # A built-in name also present in a --search-dir is ambiguous; the user must disambiguate with --config. + self._make_user_benchmark(tmp_path, name="gsm8k") # gsm8k also exists under WORKING_DIR + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr( + sys, "argv", ["gym", "eval", "prepare", "--benchmark", "gsm8k", "--search-dir", str(tmp_path)] + ) + with pytest.raises(SystemExit): + main() + err = capsys.readouterr().err + assert "ambiguous" in err + assert str(WORKING_DIR / "benchmarks" / "gsm8k" / "config.yaml") in err + assert str(tmp_path / "benchmarks" / "gsm8k" / "config.yaml") in err + + def test_search_dir_alone_emits_nothing(self, monkeypatch: MonkeyPatch, tmp_path) -> None: + # --search-dir is consumed by the selectors; on its own it is not a Hydra override. + _, overrides = _dispatch_for(monkeypatch, ["eval", "prepare", "--search-dir", str(tmp_path)]) + assert overrides == [] + + def test_did_you_mean_spans_user_dir(self, monkeypatch: MonkeyPatch, tmp_path, capsys) -> None: + self._make_user_benchmark(tmp_path) + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr( + sys, "argv", ["gym", "eval", "prepare", "--benchmark", "mybnch", "--search-dir", str(tmp_path)] + ) + with pytest.raises(SystemExit): + main() + assert "Did you mean `mybench`?" in capsys.readouterr().err From ba6dc10e35c12168e70baa74b915ec57b897d1a0 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Mon, 22 Jun 2026 09:55:19 +0200 Subject: [PATCH 18/40] feat: add gym search command Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/eval.py | 96 +++++++++++++++++++++++++++-- nemo_gym/cli/main.py | 11 ++++ tests/unit_tests/test_benchmarks.py | 82 +++++++++++++++++++++++- tests/unit_tests/test_cli_main.py | 12 ++++ 4 files changed, 195 insertions(+), 6 deletions(-) diff --git a/nemo_gym/cli/eval.py b/nemo_gym/cli/eval.py index 473083a358..9a6ce87e09 100644 --- a/nemo_gym/cli/eval.py +++ b/nemo_gym/cli/eval.py @@ -14,6 +14,7 @@ # limitations under the License. import asyncio +import difflib import importlib import json from copy import deepcopy @@ -34,8 +35,10 @@ from nemo_gym.config_types import BaseNeMoGymCLIConfig, BenchmarkDatasetConfig from nemo_gym.global_config import ( JSON_OUTPUT_KEY_NAME, + POLICY_MODEL_KEY_NAME, ROLLOUT_INDEX_KEY_NAME, TASK_INDEX_KEY_NAME, + GlobalConfigDictParser, GlobalConfigDictParserConfig, get_first_server_config_dict, get_global_config_dict, @@ -51,8 +54,66 @@ from nemo_gym.train_data_utils import TrainDataProcessor +def _fuzzy_matches(query: str, *fields: str) -> bool: + """Whether `query` fuzzily matches any of `fields`: a substring or a close difflib match (token-aware).""" + needle = query.lower() + for field in fields: + if not field: + continue + haystack = field.lower() + if needle in haystack: + return True + tokens = haystack.replace("_", " ").replace("-", " ").split() + if difflib.get_close_matches(needle, [haystack, *tokens], n=1, cutoff=0.6): + return True + return False + + +def _benchmark_extras(bench: BenchmarkConfig) -> tuple[str, list[str]]: + """Resolve a benchmark's config to its `(domain, extra search terms)`. + + `BenchmarkConfig` flattens away the resource server name, the resource server `domain`, and the + dataset names. We re-resolve the config with the same parser `BenchmarkConfig` uses (so chained + `config_paths` / `_inherit_from` are applied) and read those fields back out for the domain column + and richer `gym search` matching. + """ + initial_config_dict = OmegaConf.load(bench.path) + if POLICY_MODEL_KEY_NAME not in initial_config_dict: + initial_config_dict = OmegaConf.merge( + initial_config_dict, GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT + ) + resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config_dict) + + domain = "" + terms: list[str] = [] + for instance_name in resolved: + instance = resolved[instance_name] + if not isinstance(instance, (dict, DictConfig)): + continue + + resource_servers = instance.get("resources_servers") + if resource_servers: + terms.append(instance_name) # e.g. aime24_math_with_judge_resources_server + for rs_name, rs_config in resource_servers.items(): + terms.append(rs_name) # e.g. math_with_judge + found_domain = (rs_config or {}).get("domain") + if found_domain: + domain = str(found_domain) + + agents = instance.get("responses_api_agents") + if agents: + for agent_config in agents.values(): + for dataset in (agent_config or {}).get("datasets") or []: + if (dataset or {}).get("name"): + terms.append(dataset["name"]) + + if domain: + terms.append(domain) + return domain, terms + + def list_benchmarks() -> None: - """CLI command: list available benchmarks.""" + """CLI command: list available benchmarks, optionally filtered by a `query` (the `gym search` entry point).""" global_config_dict = get_global_config_dict( global_config_dict_parser_config=GlobalConfigDictParserConfig( initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, @@ -68,26 +129,53 @@ def list_benchmarks() -> None: benchmarks = _load_benchmarks_from_config_paths(config_paths) + # Resolve the domain + richer search terms once per benchmark, for the domain column and `gym search`. + extras = {name: _benchmark_extras(bench) for name, bench in benchmarks.items()} + + # `gym search ` reuses this command, narrowing the listing to fuzzy matches across the + # benchmark name, agent name, resource server name, dataset names, and domain. + query = global_config_dict.get("query") + if query: + benchmarks = { + name: bench + for name, bench in benchmarks.items() + if _fuzzy_matches(query, name, bench.agent_name, *extras[name][1]) + } + if global_config_dict.get(JSON_OUTPUT_KEY_NAME, False): payload = [ - {"name": name, "agent_name": bench.agent_name, "num_repeats": bench.num_repeats} + { + "name": name, + "agent_name": bench.agent_name, + "domain": extras[name][0], + "num_repeats": bench.num_repeats, + } for name, bench in benchmarks.items() ] print(json.dumps(payload)) return if not benchmarks: + if query: + rich.print(f"[yellow]No benchmarks match '{query}'.[/yellow]") + return rich.print("[yellow]No benchmarks found.[/yellow]") rich.print(f"Expected benchmarks directory: {BENCHMARKS_DIR}") return - table = Table(title=f"Available benchmarks in NeMo Gym ({len(benchmarks)})") + title = ( + f"Benchmarks matching '{query}' ({len(benchmarks)})" + if query + else f"Available benchmarks in NeMo Gym ({len(benchmarks)})" + ) + table = Table(title=title) table.add_column("Benchmark name") + table.add_column("Domain") table.add_column("Agent name") table.add_column("Num repeats") for name, bench in benchmarks.items(): - table.add_row(name, bench.agent_name, str(bench.num_repeats)) + table.add_row(name, extras[name][0], bench.agent_name, str(bench.num_repeats)) rich.print(table) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index e525166d94..5f743f5c08 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -132,6 +132,12 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: # env status). The reserved `json` config key is read centrally via nemo_gym.cli.output.emit. JSON = _bool_flag("json", "json", "Output as JSON for programmatic use.") +# Positional search query for `gym search`; surfaced to the listing command as the `query` config key. +QUERY = Flag( + register=lambda p: p.add_argument("query", metavar="QUERY", help="Substring to match against component names."), + translate_to_hydra=lambda args: [f"+query={args.query}"] if getattr(args, "query", None) else [], +) + # Asset selector flag -> (parent dir, configs subdir, default config flavor). All accept `name` or `name/flavor`, # resolving to `//[/].yaml`. A None default flavor falls back to the server name. @@ -280,6 +286,11 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "list benchmarks": Command( target="nemo_gym.cli.eval:list_benchmarks", summary="List available benchmarks.", flags=(JSON,) ), + "search": Command( + target="nemo_gym.cli.eval:list_benchmarks", + summary="Search available components (currently benchmarks) by name; like `list` filtered to a query.", + flags=(QUERY, JSON), + ), "dataset upload": Command( target=_dataset_upload, summary="Upload a prepared dataset to HF (default) or GitLab.", diff --git a/tests/unit_tests/test_benchmarks.py b/tests/unit_tests/test_benchmarks.py index a104a43490..8ac1de1806 100644 --- a/tests/unit_tests/test_benchmarks.py +++ b/tests/unit_tests/test_benchmarks.py @@ -19,7 +19,7 @@ from omegaconf import OmegaConf from yaml import safe_load -from nemo_gym.cli.eval import list_benchmarks, prepare_benchmark +from nemo_gym.cli.eval import _benchmark_extras, _fuzzy_matches, list_benchmarks, prepare_benchmark def _mock_global_config(config: dict = None): @@ -48,10 +48,11 @@ def test_json_output(self, capsys) -> None: with ( patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config({"json": True})), patch("nemo_gym.cli.eval._load_benchmarks_from_config_paths", return_value={"my_bench": bench}), + patch("nemo_gym.cli.eval._benchmark_extras", return_value=("math", ["math_with_judge"])), ): list_benchmarks() assert json.loads(capsys.readouterr().out) == [ - {"name": "my_bench", "agent_name": "my_agent", "num_repeats": 4} + {"name": "my_bench", "agent_name": "my_agent", "domain": "math", "num_repeats": 4} ] def test_json_output_empty(self, capsys) -> None: @@ -65,6 +66,83 @@ def test_json_output_empty(self, capsys) -> None: assert json.loads(capsys.readouterr().out) == [] +class TestFuzzyMatches: + def test_substring_matches(self) -> None: + assert _fuzzy_matches("math", "math_with_judge") + + def test_token_typo_matches(self) -> None: + # `aimee` is a near-miss for the `aime` token in `aime24`. + assert _fuzzy_matches("aimee", "aime24") + + def test_matches_against_agent_field(self) -> None: + assert _fuzzy_matches("judge", "aime24", "math_with_judge_agent") + + def test_skips_empty_fields(self) -> None: + assert not _fuzzy_matches("math", "", None) + + def test_no_match(self) -> None: + assert not _fuzzy_matches("zzznomatch", "aime24", "math_with_judge") + + +class TestBenchmarkExtras: + def test_resolves_domain_and_terms_from_real_config(self) -> None: + from nemo_gym.benchmarks import BENCHMARKS_DIR, BenchmarkConfig + + bench = BenchmarkConfig.from_config_path(BENCHMARKS_DIR / "aime24" / "config.yaml") + domain, terms = _benchmark_extras(bench) + + assert domain == "math" + # The resource server's inner name and the dataset name are recovered for search. + assert "math_with_judge" in terms + assert "aime24" in terms + + +class TestSearchBenchmarks: + # Map each benchmark name to the (domain, extra terms) its config would resolve to. + # As in the real resolver, `domain` is also included among the search terms. + EXTRAS = { + "aime24": ("math", ["aime24_math_resources_server", "math_with_judge", "aime24", "math"]), + "gpqa_diamond": ("science", ["gpqa_resources_server", "gpqa", "gpqa_diamond", "science"]), + } + + def _bench(self, key: str): + bench = MagicMock(agent_name="my_agent", num_repeats=1) + bench.config_key = key # let the patched _benchmark_extras find the right entry + return bench + + def _benchmarks(self) -> dict: + return {name: self._bench(name) for name in self.EXTRAS} + + def _run(self, query: str, benchmarks: dict, capsys) -> str: + with ( + patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config({"query": query})), + patch("nemo_gym.cli.eval._load_benchmarks_from_config_paths", return_value=benchmarks), + patch("nemo_gym.cli.eval._benchmark_extras", side_effect=lambda b: self.EXTRAS[b.config_key]), + ): + list_benchmarks() + return capsys.readouterr().out + + def test_query_filters_by_name(self, capsys) -> None: + out = self._run("aime", self._benchmarks(), capsys) + assert "aime24" in out + assert "gpqa" not in out + + def test_query_matches_domain(self, capsys) -> None: + # "science" only appears via gpqa's domain, not its name/agent. + out = self._run("science", self._benchmarks(), capsys) + assert "gpqa_diamond" in out + assert "aime24" not in out + + def test_query_matches_resource_server(self, capsys) -> None: + # "judge" only appears via aime24's resource server name. + out = self._run("judge", self._benchmarks(), capsys) + assert "aime24" in out + assert "gpqa_diamond" not in out + + def test_query_no_match_message(self, capsys) -> None: + assert "No benchmarks match 'zzz'" in self._run("zzz", self._benchmarks(), capsys) + + class TestPrepareBenchmark: def _make_bench_dir(self, tmp_path: Path, name: str = "fake_bench") -> tuple[Path, Path]: benchmarks_dir = tmp_path / "benchmarks" diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 0a8c59a1ef..370273a06e 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -500,6 +500,18 @@ def test_no_json_no_override(self, monkeypatch: MonkeyPatch) -> None: _, overrides = _dispatch_for(monkeypatch, ["list", "benchmarks"]) assert overrides == [] + +class TestSearch: + def test_search_routes_to_list_with_query(self, monkeypatch: MonkeyPatch) -> None: + # `gym search ` reuses the benchmarks listing, passing the query as the `query` config key. + target, overrides = _dispatch_for(monkeypatch, ["search", "math"]) + assert target == "nemo_gym.cli.eval:list_benchmarks" + assert overrides == ["+query=math"] + + def test_search_json(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["search", "math", "--json"]) + assert set(overrides) == {"+query=math", "+json=true"} + def test_version_json_dispatches_with_override(self, monkeypatch: MonkeyPatch) -> None: # `gym --version --json` is the top-level path; it still forwards +json=true to the version command. target, overrides = _dispatch_for(monkeypatch, ["--version", "--json"]) From c53e912c3bc72284638e4d52b040ac97f96ca770 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Mon, 22 Jun 2026 10:17:43 +0200 Subject: [PATCH 19/40] feat: add generic deployment config and --model-checkpoint flag Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 20 +++++++++++++- .../configs/local_vllm_model.yaml | 26 +++++++++++++++++++ tests/unit_tests/test_cli_main.py | 22 ++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 responses_api_models/local_vllm_model/configs/local_vllm_model.yaml diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 5f743f5c08..a84f0655e8 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -120,6 +120,14 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: MODEL_URL = _value_flag("model-url", "policy_base_url", "Model server base URL.") MODEL_API_KEY = _value_flag("model-api-key", "policy_api_key", "Model server API key.") +# Shared flag: checkpoint (HF id or path) to serve with a local vLLM deployment. Pair with `--model-type +# local_vllm_model`; sets the served model via `policy_model_name`. Reused by model-server commands. +MODEL_CHECKPOINT = _value_flag( + "model-checkpoint", + "policy_model_name", + "HF id or checkpoint path to serve locally (use with --model-type local_vllm_model).", +) + # Shared flag: select a single resource server by name. Reused by `env test`, `env init`, and `env packages`. RESOURCE_SERVER = Flag( register=lambda p: p.add_argument("--resource-server", metavar="NAME", help="Name of the resource server."), @@ -401,7 +409,16 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "env run": Command( target="nemo_gym.cli.env:run", summary="Start the servers.", - flags=(CONFIG, RESOURCE_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, MODEL_NAME, MODEL_URL, MODEL_API_KEY), + flags=( + CONFIG, + RESOURCE_SERVER_CONFIG, + MODEL_TYPE, + SEARCH_DIR, + MODEL_NAME, + MODEL_URL, + MODEL_API_KEY, + MODEL_CHECKPOINT, + ), ), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status.", flags=(JSON,)), "eval prepare": Command( @@ -437,6 +454,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: MODEL_NAME, MODEL_URL, MODEL_API_KEY, + MODEL_CHECKPOINT, _value_flag("temperature", "responses_create_params.temperature", "Sampling temperature."), _value_flag("top-p", "responses_create_params.top_p", "Nucleus sampling top-p."), _value_flag("max-output-tokens", "responses_create_params.max_output_tokens", "Maximum output tokens."), diff --git a/responses_api_models/local_vllm_model/configs/local_vllm_model.yaml b/responses_api_models/local_vllm_model/configs/local_vllm_model.yaml new file mode 100644 index 0000000000..d0712f6b36 --- /dev/null +++ b/responses_api_models/local_vllm_model/configs/local_vllm_model.yaml @@ -0,0 +1,26 @@ +# Generic local vLLM deployment. Serves the checkpoint passed via `--model-checkpoint ` +# (i.e. `policy_model_name`). Set parallelism to match your node before running. +# +# Example: +# gym eval run --benchmark aime24 --model-type local_vllm_model --model-checkpoint Qwen/Qwen3-30B-A3B-Instruct-2507 +policy_model: + responses_api_models: + local_vllm_model: + entrypoint: app.py + model: ${policy_model_name} + return_token_id_information: false + uses_reasoning_parser: false + + # If your model is downloaded at ~/.cache/huggingface/hub/models----, set hf_home to ~/.cache/huggingface. + hf_home: null + + vllm_serve_env_vars: {} + + # vLLM parallelism is sensitive; tensor_parallel_size * pipeline_parallel_size GPUs are used per model instance. + # Override per node, e.g. `++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.tensor_parallel_size=8`. + vllm_serve_kwargs: + data_parallel_size: 1 + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + trust_remote_code: true + gpu_memory_utilization: 0.9 diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 370273a06e..0f6992ca74 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -465,6 +465,28 @@ def test_model_flags(self, monkeypatch: MonkeyPatch) -> None: } +class TestModelCheckpointFlag: + def test_sets_served_model_for_eval_run(self, monkeypatch: MonkeyPatch) -> None: + # --model-checkpoint sets the served model (policy_model_name); the local vLLM config is selected separately. + target, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--model-checkpoint", "Qwen/Qwen3-8B"]) + assert target == "nemo_gym.cli.eval:e2e_rollout_collection" + assert overrides == ["+policy_model_name=Qwen/Qwen3-8B"] + + def test_available_on_env_run(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--model-checkpoint", "/ckpt/path"]) + assert overrides == ["+policy_model_name=/ckpt/path"] + + def test_local_vllm_deployment_flow(self, monkeypatch: MonkeyPatch) -> None: + # The intended deployment invocation: select the local vLLM server type and pass the checkpoint to serve. + _, overrides = _dispatch_for( + monkeypatch, + ["eval", "run", "--model-type", "local_vllm_model", "--model-checkpoint", "Qwen/Qwen3-8B"], + ) + paths, others = _split_overrides(overrides) + assert paths == {str(WORKING_DIR / "responses_api_models/local_vllm_model/configs/local_vllm_model.yaml")} + assert others == {"+policy_model_name=Qwen/Qwen3-8B"} + + class TestEnvInitFlags: def test_resource_server_translates_to_entrypoint(self, monkeypatch: MonkeyPatch) -> None: target, overrides = _dispatch_for(monkeypatch, ["env", "init", "--resource-server", "my_server"]) From 4b97e8a32772ba4cd6c5379b4c2926831f5594f5 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Mon, 22 Jun 2026 10:23:17 +0200 Subject: [PATCH 20/40] fix: unify rendundant --model-name and --model-checkpoint flags into a single --model flag Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 31 ++++++------------- .../configs/local_vllm_model.yaml | 4 +-- tests/unit_tests/test_cli_main.py | 27 +++++++--------- 3 files changed, 24 insertions(+), 38 deletions(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index a84f0655e8..0eace3b678 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -116,17 +116,16 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: ) # Shared model-server flags. Reused by commands that spin up / target a model server (`eval run`, `env run`). -MODEL_NAME = _value_flag("model-name", "policy_model_name", "Model name.") -MODEL_URL = _value_flag("model-url", "policy_base_url", "Model server base URL.") -MODEL_API_KEY = _value_flag("model-api-key", "policy_api_key", "Model server API key.") - -# Shared flag: checkpoint (HF id or path) to serve with a local vLLM deployment. Pair with `--model-type -# local_vllm_model`; sets the served model via `policy_model_name`. Reused by model-server commands. -MODEL_CHECKPOINT = _value_flag( - "model-checkpoint", +# --model is the served model identifier across all backends: an API model name, an HF id, or a local checkpoint +# path, interpreted per --model-type (e.g. a path/HF id to serve with local_vllm_model). +MODEL = _value_flag( + "model", "policy_model_name", - "HF id or checkpoint path to serve locally (use with --model-type local_vllm_model).", + "Model name, HF id, or local checkpoint path (interpreted per --model-type).", + aliases=("-m",), ) +MODEL_URL = _value_flag("model-url", "policy_base_url", "Model server base URL.") +MODEL_API_KEY = _value_flag("model-api-key", "policy_api_key", "Model server API key.") # Shared flag: select a single resource server by name. Reused by `env test`, `env init`, and `env packages`. RESOURCE_SERVER = Flag( @@ -409,16 +408,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "env run": Command( target="nemo_gym.cli.env:run", summary="Start the servers.", - flags=( - CONFIG, - RESOURCE_SERVER_CONFIG, - MODEL_TYPE, - SEARCH_DIR, - MODEL_NAME, - MODEL_URL, - MODEL_API_KEY, - MODEL_CHECKPOINT, - ), + flags=(CONFIG, RESOURCE_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, MODEL, MODEL_URL, MODEL_API_KEY), ), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status.", flags=(JSON,)), "eval prepare": Command( @@ -451,10 +441,9 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: _value_flag("prompt-config", "prompt_config", "Prompt template YAML to apply."), _value_flag("concurrency", "num_samples_in_parallel", "Maximum number of concurrent samples."), _value_flag("split", "split", "Dataset split to use (train, validation, or benchmark)."), - MODEL_NAME, + MODEL, MODEL_URL, MODEL_API_KEY, - MODEL_CHECKPOINT, _value_flag("temperature", "responses_create_params.temperature", "Sampling temperature."), _value_flag("top-p", "responses_create_params.top_p", "Nucleus sampling top-p."), _value_flag("max-output-tokens", "responses_create_params.max_output_tokens", "Maximum output tokens."), diff --git a/responses_api_models/local_vllm_model/configs/local_vllm_model.yaml b/responses_api_models/local_vllm_model/configs/local_vllm_model.yaml index d0712f6b36..09ff537d96 100644 --- a/responses_api_models/local_vllm_model/configs/local_vllm_model.yaml +++ b/responses_api_models/local_vllm_model/configs/local_vllm_model.yaml @@ -1,8 +1,8 @@ -# Generic local vLLM deployment. Serves the checkpoint passed via `--model-checkpoint ` +# Generic local vLLM deployment. Serves the checkpoint passed via `--model ` # (i.e. `policy_model_name`). Set parallelism to match your node before running. # # Example: -# gym eval run --benchmark aime24 --model-type local_vllm_model --model-checkpoint Qwen/Qwen3-30B-A3B-Instruct-2507 +# gym eval run --benchmark aime24 --model-type local_vllm_model --model Qwen/Qwen3-30B-A3B-Instruct-2507 policy_model: responses_api_models: local_vllm_model: diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 0f6992ca74..9f1f5cb7e4 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -139,7 +139,8 @@ class TestEvalRunFlags: (["--concurrency", "10"], "+num_samples_in_parallel=10"), (["--prompt-config", "p.yaml"], "+prompt_config=p.yaml"), (["--split", "benchmark"], "+split=benchmark"), - (["--model-name", "openai/gpt-oss-120b"], "+policy_model_name=openai/gpt-oss-120b"), + (["--model", "openai/gpt-oss-120b"], "+policy_model_name=openai/gpt-oss-120b"), + (["-m", "openai/gpt-oss-120b"], "+policy_model_name=openai/gpt-oss-120b"), (["--model-url", "http://0.0.0.0:10240/v1"], "+policy_base_url=http://0.0.0.0:10240/v1"), (["--model-api-key", "sk-your-api-key"], "+policy_api_key=sk-your-api-key"), (["--temperature", "1.0"], "+responses_create_params.temperature=1.0"), @@ -202,7 +203,7 @@ def test_readme_model_and_sampling_example(self, monkeypatch: MonkeyPatch) -> No [ "eval", "run", - "--model-name", + "--model", "openai/gpt-oss-120b", "--model-url", "http://0.0.0.0:10240/v1", @@ -448,7 +449,7 @@ def test_model_flags(self, monkeypatch: MonkeyPatch) -> None: "run", "--config", "c.yaml", - "--model-name", + "--model", "gpt", "--model-url", "http://x", @@ -465,27 +466,23 @@ def test_model_flags(self, monkeypatch: MonkeyPatch) -> None: } -class TestModelCheckpointFlag: - def test_sets_served_model_for_eval_run(self, monkeypatch: MonkeyPatch) -> None: - # --model-checkpoint sets the served model (policy_model_name); the local vLLM config is selected separately. - target, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--model-checkpoint", "Qwen/Qwen3-8B"]) - assert target == "nemo_gym.cli.eval:e2e_rollout_collection" - assert overrides == ["+policy_model_name=Qwen/Qwen3-8B"] - - def test_available_on_env_run(self, monkeypatch: MonkeyPatch) -> None: - _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--model-checkpoint", "/ckpt/path"]) - assert overrides == ["+policy_model_name=/ckpt/path"] +class TestModelFlag: + """`--model` is the single served-model identifier (name, HF id, or local checkpoint path) for any backend.""" def test_local_vllm_deployment_flow(self, monkeypatch: MonkeyPatch) -> None: - # The intended deployment invocation: select the local vLLM server type and pass the checkpoint to serve. + # The deployment invocation: select the local vLLM server type and pass the checkpoint to serve via --model. _, overrides = _dispatch_for( monkeypatch, - ["eval", "run", "--model-type", "local_vllm_model", "--model-checkpoint", "Qwen/Qwen3-8B"], + ["eval", "run", "--model-type", "local_vllm_model", "--model", "Qwen/Qwen3-8B"], ) paths, others = _split_overrides(overrides) assert paths == {str(WORKING_DIR / "responses_api_models/local_vllm_model/configs/local_vllm_model.yaml")} assert others == {"+policy_model_name=Qwen/Qwen3-8B"} + def test_short_alias_on_env_run(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "run", "-m", "/ckpt/path"]) + assert overrides == ["+policy_model_name=/ckpt/path"] + class TestEnvInitFlags: def test_resource_server_translates_to_entrypoint(self, monkeypatch: MonkeyPatch) -> None: From c0747bd27d72d51ebf69aa0923d524725f5b698f Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Mon, 22 Jun 2026 13:43:01 +0200 Subject: [PATCH 21/40] feat: add --benchmark flag to gym env run command Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 2 +- tests/unit_tests/test_cli_main.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 0eace3b678..d935a335ae 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -408,7 +408,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "env run": Command( target="nemo_gym.cli.env:run", summary="Start the servers.", - flags=(CONFIG, RESOURCE_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, MODEL, MODEL_URL, MODEL_API_KEY), + flags=(CONFIG, BENCHMARK, RESOURCE_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, MODEL, MODEL_URL, MODEL_API_KEY), ), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status.", flags=(JSON,)), "eval prepare": Command( diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 9f1f5cb7e4..bb57b44984 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -605,6 +605,8 @@ class TestAssetSelectors: (["eval", "prepare", "--benchmark", "gsm8k"], "benchmarks/gsm8k/config.yaml"), # benchmarks/aime25-x/README.md: ng_prepare_benchmark "+config_paths=[benchmarks/aime25-x/config.yaml]" (["eval", "prepare", "--benchmark", "aime25-x"], "benchmarks/aime25-x/config.yaml"), + # benchmarks/gpqa/README.md: ng_run "+config_paths=[benchmarks/gpqa/config.yaml]" (start a benchmark's servers) + (["env", "run", "--benchmark", "gpqa"], "benchmarks/gpqa/config.yaml"), # README.md / quickstart.mdx: resources_servers/mcqa/configs/mcqa.yaml (["env", "run", "--resource-server", "mcqa"], "resources_servers/mcqa/configs/mcqa.yaml"), # model-server/vllm.mdx: resources_servers/example_multi_step/configs/example_multi_step.yaml From efed90130d2f7e26b107549ad31878e7cdd8adbc Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Mon, 22 Jun 2026 14:05:46 +0200 Subject: [PATCH 22/40] chore: migrate docs, skills and READMEs to use new cli Signed-off-by: Marta Stepniewska-Dziubinska --- .claude/skills/add-benchmark/SKILL.md | 60 +- .../add-benchmark/references/patterns.md | 24 +- .../references/error-profiles.md | 4 +- .../references/request-boundary-visibility.md | 2 +- .../skills/nemo-gym-reward-profiling/SKILL.md | 12 +- .../references/output-format.md | 4 +- .../references/quick-start.md | 40 +- .../references/error-profiles.md | 4 +- .../references/request-boundary-visibility.md | 2 +- .../skills/nemo-gym-reward-profiling/SKILL.md | 14 +- .../references/output-format.md | 4 +- .../references/quick-start.md | 40 +- CLAUDE.md | 18 +- README.md | 21 +- benchmarks/aalcr/README.md | 10 +- benchmarks/aime24-x/README.md | 21 +- benchmarks/aime25-x/README.md | 23 +- benchmarks/aime26/README.md | 17 +- benchmarks/answer-judge/README.md | 18 +- benchmarks/apex_shortlist/README.md | 18 +- benchmarks/arena_hard/README.md | 22 +- benchmarks/arena_hard_v2/README.md | 20 +- benchmarks/asr_leaderboard/README.md | 16 +- benchmarks/bigcodebench/README.md | 24 +- benchmarks/birdbench/README.md | 18 +- benchmarks/browsecomp/README.md | 22 +- benchmarks/finance_sec_search/README.md | 18 +- benchmarks/flores200/README.md | 22 +- benchmarks/frontierscience_olympiad/README.md | 20 +- benchmarks/gdpval/README.md | 30 +- benchmarks/global-piqa/README.md | 18 +- benchmarks/gpqa-x/README.md | 23 +- benchmarks/gpqa/README.md | 13 +- benchmarks/graphwalks/README.md | 28 +- benchmarks/gsm8k/README.md | 18 +- benchmarks/hendrycks_math/README.md | 18 +- benchmarks/hle/README.md | 22 +- benchmarks/hmmt_feb25/README.md | 23 +- benchmarks/hmmt_nov25/README.md | 26 +- benchmarks/hotpotqa_closedbook/README.md | 17 +- benchmarks/human_eval/README.md | 24 +- benchmarks/human_eval_infilling/README.md | 38 +- benchmarks/ifbench/README.md | 21 +- benchmarks/ifeval/README.md | 18 +- benchmarks/imo_answerbench/README.md | 19 +- benchmarks/imo_gradingbench/README.md | 20 +- benchmarks/imo_proofbench/README.md | 22 +- benchmarks/ioi/README.md | 28 +- benchmarks/labbench2_vlm/README.md | 52 +- benchmarks/librispeech_pc/README.md | 18 +- benchmarks/livecodebench-x/README.md | 34 +- benchmarks/longbench_v2/README.md | 34 +- benchmarks/longcodebench/README.md | 34 +- benchmarks/m_arena_hard/README.md | 20 +- benchmarks/m_arena_hard_v2/README.md | 20 +- benchmarks/math-500/README.md | 18 +- benchmarks/mbpp/README.md | 24 +- benchmarks/minif2f/README.md | 23 +- benchmarks/mmlu-redux/README.md | 18 +- benchmarks/mmlu/README.md | 18 +- benchmarks/mmlu_pro/README.md | 13 +- benchmarks/mmlu_prox/README.md | 13 +- benchmarks/mmmlu/README.md | 18 +- benchmarks/mobench/README.md | 23 +- benchmarks/mrcr/README.md | 40 +- benchmarks/musan/README.md | 16 +- benchmarks/nemotron_3_ultra/README.md | 24 +- benchmarks/numb3rs/README.md | 16 +- benchmarks/omniscience/README.md | 9 +- benchmarks/physics/README.md | 18 +- benchmarks/polymath/README.md | 18 +- benchmarks/proof-arena-judge/README.md | 18 +- benchmarks/proof_bench_judge/README.md | 18 +- benchmarks/proofnet/README.md | 23 +- benchmarks/putnam_bench/README.md | 23 +- benchmarks/ruler/README.md | 24 +- benchmarks/simpleqa/README.md | 20 +- benchmarks/speed-bench/README.md | 18 +- benchmarks/spider2_lite/README.md | 24 +- benchmarks/supergpqa/README.md | 18 +- benchmarks/ugphysics/README.md | 24 +- benchmarks/wmt24pp/README.md | 24 +- benchmarks/xstest/README.md | 24 +- environments/workplace_assistant/README.md | 15 +- .../pages/contribute/development-setup.mdx | 12 +- .../environments/new-environment.mdx | 4 +- .../pages/data/download-huggingface.mdx | 56 +- fern/versions/latest/pages/data/index.mdx | 36 +- .../latest/pages/data/prepare-validate.mdx | 56 +- .../latest/pages/data/prompt-config.mdx | 47 +- .../aggregate-metrics.mdx | 2 +- .../integrate-external-environments.mdx | 2 +- .../single-step-environment.mdx | 32 +- .../verification-patterns/llm-as-judge.mdx | 21 +- .../latest/pages/get-started/installation.mdx | 2 +- .../latest/pages/get-started/quickstart.mdx | 29 +- .../engineering-notes/system-design.mdx | 14 +- .../latest/pages/model-server/vllm.mdx | 25 +- .../latest/pages/reference/cli-commands.mdx | 840 ++++++++++-------- .../latest/pages/reference/configuration.mdx | 19 +- fern/versions/latest/pages/reference/faq.mdx | 137 +-- .../multi-environment-training.mdx | 26 +- .../training-tutorials/nemo-rl-grpo/setup.mdx | 15 +- .../offline-training-w-rollouts.mdx | 7 +- .../latest/pages/training-tutorials/verl.mdx | 14 +- .../pages/troubleshooting/configuration.mdx | 19 +- resources_servers/abstention/README.md | 16 +- resources_servers/arc_agi/README.md | 14 +- resources_servers/arena_judge/README.md | 16 +- resources_servers/asr_with_pc/README.md | 16 +- resources_servers/aviary/README.md | 41 +- resources_servers/bigcodebench/README.md | 16 +- resources_servers/bird_sql/README.md | 16 +- resources_servers/blackjack/README.md | 12 +- .../browsecomp_advanced_harness/README.md | 7 +- resources_servers/calendar/README.md | 16 +- resources_servers/circle_click/README.md | 10 +- resources_servers/circle_count/README.md | 10 +- resources_servers/code_fim/README.md | 16 +- resources_servers/code_gen/README.md | 15 +- .../competitive_coding_challenges/README.md | 16 +- resources_servers/cvdp/README.md | 27 +- .../equivalence_llm_judge/README.md | 7 +- resources_servers/ether0/README.md | 12 +- resources_servers/evalplus/README.md | 16 +- .../example_multi_turn_gymnasium/README.md | 12 +- .../finance_sec_search/README.md | 36 +- .../format_verification/README.md | 62 +- .../frontierscience_judge/README.md | 16 +- resources_servers/gdpval/README.md | 9 +- resources_servers/google_search/README.md | 15 +- resources_servers/gpqa_diamond/README.md | 233 ++--- resources_servers/graphwalks/README.md | 12 +- resources_servers/grl_sokoban/README.md | 17 +- resources_servers/grl_tetris/README.md | 17 +- resources_servers/hotpotqa_qa/README.md | 16 +- resources_servers/ifbench/README.md | 23 +- resources_servers/imo_gradingbench/README.md | 16 +- .../imo_proofbench_judge/README.md | 16 +- .../instruction_following/README.md | 34 +- resources_servers/inverse_if/README.md | 50 +- resources_servers/inverse_if/data/README.md | 10 +- .../jailbreak_detection/README.md | 17 +- resources_servers/labbench2_vlm/README.md | 61 +- resources_servers/longmt_eval/README.md | 28 +- .../math_advanced_calculations/README.md | 16 +- .../math_proof_judgement/README.md | 16 +- .../math_with_autograder/README.md | 16 +- resources_servers/math_with_code/README.md | 22 +- resources_servers/math_with_judge/README.md | 38 +- resources_servers/mcqa/README.md | 29 +- resources_servers/mrcr/README.md | 12 +- resources_servers/multichallenge/README.md | 50 +- .../multichallenge/data/README.md | 10 +- resources_servers/newton_bench/README.md | 17 +- resources_servers/ns_tools/README.md | 30 +- resources_servers/openenv/README.md | 60 +- resources_servers/physics_judge/README.md | 18 +- resources_servers/polymath/README.md | 16 +- resources_servers/rdkit_chemistry/README.md | 17 +- resources_servers/reasoning_gym/README.md | 14 +- resources_servers/simpleqa/README.md | 16 +- .../README.md | 14 +- resources_servers/speed_bench/README.md | 20 +- resources_servers/spider2_lite/README.md | 6 +- resources_servers/structeval/README.md | 20 +- .../structured_outputs/README.md | 85 +- resources_servers/swerl_gen/README.md | 23 +- resources_servers/swerl_llm_judge/README.md | 26 +- resources_servers/tavily_search/README.md | 33 +- resources_servers/terminus_judge/README.md | 14 +- .../terminus_judge/scripts/README.md | 2 +- resources_servers/text_to_sql/README.md | 14 +- resources_servers/ugphysics_judge/README.md | 18 +- resources_servers/verifif/README.md | 14 +- resources_servers/vlm_eval_kit/README.md | 21 +- resources_servers/wmt_translation/README.md | 18 +- .../workplace_assistant/README.md | 15 +- resources_servers/xlam_fc/README.md | 16 +- resources_servers/xstest/README.md | 18 +- .../claude_code_agent/README.md | 12 +- responses_api_agents/harbor_agent/README.md | 36 +- responses_api_agents/hermes_agent/README.md | 14 +- .../langgraph_agent/README.md | 14 +- responses_api_agents/mini_swe_agent/README.md | 24 +- responses_api_agents/stirrup_agent/README.md | 32 +- responses_api_agents/swe_agents/README.md | 29 +- responses_api_agents/tau2/README.md | 6 +- .../verifiers_agent/README.md | 30 +- .../azure_openai_model/README.md | 19 +- .../local_vllm_model/README.md | 8 +- .../local_vllm_model_proxy/README.md | 6 +- 192 files changed, 2724 insertions(+), 2452 deletions(-) diff --git a/.claude/skills/add-benchmark/SKILL.md b/.claude/skills/add-benchmark/SKILL.md index 385666e384..3ed1460d87 100644 --- a/.claude/skills/add-benchmark/SKILL.md +++ b/.claude/skills/add-benchmark/SKILL.md @@ -32,10 +32,10 @@ Before starting, determine which type of benchmark you're adding: ### Step 1: Scaffold the server -Run `ng_init_resources_server` to generate the directory structure: +Run `gym env init` to generate the directory structure: ```bash -ng_init_resources_server +entrypoint=resources_servers/my_benchmark +gym env init --resource-server my_benchmark ``` This creates: @@ -77,10 +77,10 @@ Convert your source dataset to Gym JSONL format. Each line must have `responses_ **`train`/`validation` datasets**: Upload to the GitLab dataset registry — these must NOT be committed to git. ```bash -ng_upload_dataset_to_gitlab \ - +dataset_name=my_benchmark \ - +version=0.0.1 \ - +input_jsonl_fpath=resources_servers/my_benchmark/data/my_dataset.jsonl +gym dataset upload --storage gitlab \ + --name my_benchmark \ + --revision 0.0.1 \ + --input resources_servers/my_benchmark/data/my_dataset.jsonl ``` Requires MLflow credentials in `env.yaml` (or passed via CLI): @@ -94,12 +94,16 @@ mlflow_tracking_token: **Validate** your data: ```bash # Validate example data (for PR submission) -ng_prepare_data "+config_paths=[resources_servers/my_benchmark/configs/my_benchmark.yaml]" \ - +output_dirpath=/tmp/prepare +mode=example_validation +gym dataset collate --config resources_servers/my_benchmark/configs/my_benchmark.yaml \ + --output-dir /tmp/prepare \ + --mode example_validation # Download and prepare train/validation from GitLab -ng_prepare_data "+config_paths=[resources_servers/my_benchmark/configs/my_benchmark.yaml]" \ - +output_dirpath=data/my_benchmark +mode=train_preparation +should_download=true +data_source=gitlab +gym dataset collate --config resources_servers/my_benchmark/configs/my_benchmark.yaml \ + --output-dir data/my_benchmark \ + --mode train_preparation \ + --download \ + +data_source=gitlab ``` ### Step 3: Implement verify() @@ -162,7 +166,7 @@ Both fields must coexist: `jsonl_fpath` is the local download destination, `gitl ```bash # Run server tests (creates isolated .venv, slow on first run) -ng_test +entrypoint=resources_servers/my_benchmark +gym env test --resource-server my_benchmark # Run core library tests to check nothing broke pytest tests/unit_tests/ -x @@ -174,14 +178,18 @@ Test coverage must be >= 95%. Write tests for: verify pass, verify fail (wrong o ```bash # Start servers -ng_run "+config_paths=[resources_servers/my_benchmark/configs/my_benchmark.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env run \ + --config resources_servers/my_benchmark/configs/my_benchmark.yaml \ + --model-type openai_model # Quick test with example data -ng_collect_rollouts +agent_name=my_benchmark_simple_agent \ - +input_jsonl_fpath=resources_servers/my_benchmark/data/example.jsonl \ - +output_jsonl_fpath=results/example_rollouts.jsonl \ - +num_repeats=1 \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent my_benchmark_simple_agent \ + --input resources_servers/my_benchmark/data/example.jsonl \ + --output results/example_rollouts.jsonl \ + --num-repeats 1 \ + --max-output-tokens 16384 \ + --temperature 1.0 # Inspect results ``` @@ -196,17 +204,17 @@ Run against multiple models to validate correctness. Recommended suite: ```bash # Collect rollouts -ng_collect_rollouts +agent_name=my_benchmark_simple_agent \ - +input_jsonl_fpath=resources_servers/my_benchmark/data/my_dataset.jsonl \ - +output_jsonl_fpath=results/rollouts.jsonl \ - +num_repeats=5 \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent my_benchmark_simple_agent \ + --input resources_servers/my_benchmark/data/my_dataset.jsonl \ + --output results/rollouts.jsonl \ + --num-repeats 5 \ + --max-output-tokens 16384 \ + --temperature 1.0 # Compute per-task pass rates -ng_reward_profile +input_jsonl_fpath=resources_servers/my_benchmark/data/my_dataset.jsonl \ - +rollouts_jsonl_fpath=results/rollouts.jsonl \ - +output_jsonl_fpath=results/profiled.jsonl \ - +pass_threshold=1.0 +gym eval profile --inputs resources_servers/my_benchmark/data/my_dataset.jsonl \ + --rollouts results/rollouts.jsonl # Aggregate metrics (pass@1 = avg_reward, pass@k from max_reward) python scripts/print_aggregate_results.py +jsonl_fpath=results/profiled.jsonl diff --git a/.claude/skills/add-benchmark/references/patterns.md b/.claude/skills/add-benchmark/references/patterns.md index 68d3668b9d..4c3a2c0fc9 100644 --- a/.claude/skills/add-benchmark/references/patterns.md +++ b/.claude/skills/add-benchmark/references/patterns.md @@ -546,7 +546,7 @@ Externalize system prompts to text files, pass via `--system-prompt` argument. M ### `data/.gitignore` default patterns -Generated by `ng_init_resources_server`: +Generated by `gym env init`: ``` *train.jsonl *validation.jsonl @@ -562,10 +562,10 @@ If your filename doesn't match (e.g. `my_eval.jsonl`), add a custom pattern (e.g 1. **Generate** the JSONL file using your conversion script (in the source repo) 2. **Upload** to GitLab dataset registry: ```bash - ng_upload_dataset_to_gitlab \ - +dataset_name=my_benchmark \ - +version=0.0.1 \ - +input_jsonl_fpath=resources_servers/my_benchmark/data/my_dataset.jsonl + gym dataset upload --storage gitlab \ + --name my_benchmark \ + --revision 0.0.1 \ + --input resources_servers/my_benchmark/data/my_dataset.jsonl ``` 3. **Add `gitlab_identifier`** to the dataset entry in YAML config: ```yaml @@ -591,18 +591,22 @@ mlflow_tracking_token: The tracking URI format is `https:///api/v4/projects//ml/mlflow`. -### Verification with `ng_prepare_data` +### Verification with `gym dataset collate` Validate example data (for PR submission): ```bash -ng_prepare_data "+config_paths=[resources_servers/my_benchmark/configs/my_benchmark.yaml]" \ - +output_dirpath=/tmp/prepare +mode=example_validation +gym dataset collate --config resources_servers/my_benchmark/configs/my_benchmark.yaml \ + --output-dir /tmp/prepare \ + --mode example_validation ``` Download and prepare train/validation from GitLab: ```bash -ng_prepare_data "+config_paths=[resources_servers/my_benchmark/configs/my_benchmark.yaml]" \ - +output_dirpath=data/my_benchmark +mode=train_preparation +should_download=true +data_source=gitlab +gym dataset collate --config resources_servers/my_benchmark/configs/my_benchmark.yaml \ + --output-dir data/my_benchmark \ + --mode train_preparation \ + --download \ + +data_source=gitlab ``` --- diff --git a/.claude/skills/nemo-gym-debugging/references/error-profiles.md b/.claude/skills/nemo-gym-debugging/references/error-profiles.md index 666b9eb9d4..7c9d540f65 100644 --- a/.claude/skills/nemo-gym-debugging/references/error-profiles.md +++ b/.claude/skills/nemo-gym-debugging/references/error-profiles.md @@ -113,7 +113,7 @@ Next actions: Symptoms: -- `ng_run` or `ng_collect_rollouts` fails before launching servers +- `gym env run` or `gym eval run --no-serve` fails before launching servers - errors mention missing config keys, unknown overrides, or interpolation failures - wrong agent/resource server starts despite expected data @@ -134,7 +134,7 @@ Evidence to collect: Next actions: -- reduce to a minimal `ng_run "+config_paths=[...]"` command +- reduce to a minimal `gym env run --config ...` command - add overrides back one at a time - trust target checkout code over copied command templates diff --git a/.claude/skills/nemo-gym-debugging/references/request-boundary-visibility.md b/.claude/skills/nemo-gym-debugging/references/request-boundary-visibility.md index c992750f40..0b83647324 100644 --- a/.claude/skills/nemo-gym-debugging/references/request-boundary-visibility.md +++ b/.claude/skills/nemo-gym-debugging/references/request-boundary-visibility.md @@ -9,7 +9,7 @@ This is an escalation ladder. Do not patch agents, resource servers, model adapt 1. Enable Gym's existing request debug flag: ```bash - ng_run '+config_paths=[...]' ++global_aiohttp_client_request_debug=True + gym env run --config ... ++global_aiohttp_client_request_debug=True ``` 2. Re-run the smallest failing case and capture stdout/stderr. diff --git a/.claude/skills/nemo-gym-reward-profiling/SKILL.md b/.claude/skills/nemo-gym-reward-profiling/SKILL.md index 1d177d087f..ab7067698d 100644 --- a/.claude/skills/nemo-gym-reward-profiling/SKILL.md +++ b/.claude/skills/nemo-gym-reward-profiling/SKILL.md @@ -2,7 +2,7 @@ name: nemo-gym-reward-profiling description: >- Use to help users get started with Nemo Gym reward profiling. Covers the basic - ng_run, ng_collect_rollouts, and ng_reward_profile workflow, repeated rollouts, + gym env run, gym eval run, and gym eval profile workflow, repeated rollouts, materialized inputs, rollout JSONL artifacts, task and rollout identity, output inspection, partial profiling, and rollout_infos. For failed jobs, prefer nemo-gym-debugging. @@ -14,16 +14,16 @@ description: >- Use this skill when the user wants to run, understand, or lightly modify Nemo Gym reward profiling. Keep the answer oriented around the normal workflow: -`ng_run` starts model/resource servers, `ng_collect_rollouts` writes rollout artifacts, and `ng_reward_profile` generates profiling output from those artifacts. +`gym env run` starts model/resource servers, `gym eval run --no-serve` writes rollout artifacts, and `gym eval profile` generates profiling output from those artifacts. If the user is primarily debugging a failed job or stack trace, use the `nemo-gym-debugging` skill first. ## Basic Workflow 1. Identify the environment config paths and input JSONL. -2. Start Gym servers with `ng_run`. -3. Collect rollouts with `ng_collect_rollouts`; this writes `rollouts.jsonl` and `*_materialized_inputs.jsonl`. -4. Run `ng_reward_profile` on the materialized inputs and rollout JSONL to generate `*_reward_profiling.jsonl`. +2. Start Gym servers with `gym env run`. +3. Collect rollouts with `gym eval run --no-serve`; this writes `rollouts.jsonl` and `*_materialized_inputs.jsonl`. +4. Run `gym eval profile` on the materialized inputs and rollout JSONL to generate `*_reward_profiling.jsonl`. 5. Inspect line counts and profile rows. Repeated rollouts are the main profiling lever. `num_repeats=1` is valid, but per-task averages and variance are only meaningful with multiple rollouts per task. @@ -48,6 +48,6 @@ Load references only when the user needs that detail: ## Practical Defaults -- Treat `ng_reward_profile` as the reward profiling step; rollout collection does not write reward profile files. +- Treat `gym eval profile` as the reward profiling step; rollout collection does not write reward profile files. - Run strict profiling by default. If rollout collection stopped early, use `++allow_partial_rollouts=True` to profile completed rollouts and drop original input rows with no completed rollout. - Trust the target checkout's CLI help and `nemo_gym/reward_profile.py` over memory if flags differ. diff --git a/.claude/skills/nemo-gym-reward-profiling/references/output-format.md b/.claude/skills/nemo-gym-reward-profiling/references/output-format.md index 71ab7b572d..23ab8dbfd3 100644 --- a/.claude/skills/nemo-gym-reward-profiling/references/output-format.md +++ b/.claude/skills/nemo-gym-reward-profiling/references/output-format.md @@ -6,7 +6,7 @@ Reward profiling is built around joining materialized inputs with rollout result - `*_materialized_inputs.jsonl`: expanded inputs after repeat expansion. Each row should have `_ng_task_index` and `_ng_rollout_index`. - `rollouts.jsonl`: completed rollout results. Each row should have matching `_ng_task_index` and `_ng_rollout_index`. -- `*_reward_profiling.jsonl`: task-level summaries produced by `ng_reward_profile`. +- `*_reward_profiling.jsonl`: task-level summaries produced by `gym eval profile`. - `*_agent_metrics.json`: agent/global aggregate metrics. ## Task Profile Rows @@ -36,7 +36,7 @@ Full model responses stay in `rollouts.jsonl`. Join back to full rows with `(_ng ## Partial Profiles -Strict profiling is the default. If materialized inputs and rollout results do not have the same rollout keys, `ng_reward_profile` fails and suggests: +Strict profiling is the default. If materialized inputs and rollout results do not have the same rollout keys, `gym eval profile` fails and suggests: ```bash ++allow_partial_rollouts=True diff --git a/.claude/skills/nemo-gym-reward-profiling/references/quick-start.md b/.claude/skills/nemo-gym-reward-profiling/references/quick-start.md index 0e2c80838b..9626f30605 100644 --- a/.claude/skills/nemo-gym-reward-profiling/references/quick-start.md +++ b/.claude/skills/nemo-gym-reward-profiling/references/quick-start.md @@ -5,8 +5,6 @@ Substitute environment-specific config paths, input data, model endpoint, and ou ## Minimal Flow ```bash -CONFIG_PATHS="your_model_config_paths,your_env_config_paths" - POLICY_MODEL_NAME="your_policy_model_name" POLICY_BASE_URL="your_policy_base_url" POLICY_ENDPOINT_KEY="your_policy_endpoint_key" @@ -19,41 +17,43 @@ AGENT_NAME="your_agent_name" NUM_REPEATS=2 NUM_SAMPLES_IN_PARALLEL=8 -ng_run "+config_paths=[$CONFIG_PATHS]" \ - +policy_model_name="$POLICY_MODEL_NAME" \ - +policy_base_url="$POLICY_BASE_URL" \ - +policy_api_key="$POLICY_ENDPOINT_KEY" & +gym env run \ + --config your_model_config_paths \ + --config your_env_config_paths \ + --model "$POLICY_MODEL_NAME" \ + --model-url "$POLICY_BASE_URL" \ + --model-api-key "$POLICY_ENDPOINT_KEY" & NG_RUN_PID=$! trap 'kill "$NG_RUN_PID" 2>/dev/null || true' EXIT ./scripts/wait_for_servers.sh "$NG_RUN_PID" agent_args=() if [[ -n "$AGENT_NAME" ]]; then - agent_args=(+agent_name="$AGENT_NAME") + agent_args=(--agent "$AGENT_NAME") fi -ng_collect_rollouts \ +gym eval run --no-serve \ "${agent_args[@]}" \ - +input_jsonl_fpath="$DATA_JSONL" \ - +output_jsonl_fpath="$ROLLOUTS_JSONL" \ - +num_repeats="$NUM_REPEATS" \ - +num_samples_in_parallel="$NUM_SAMPLES_IN_PARALLEL" - -ng_reward_profile \ - ++materialized_inputs_jsonl_fpath="$MATERIALIZED_JSONL" \ - ++rollouts_jsonl_fpath="$ROLLOUTS_JSONL" + --input "$DATA_JSONL" \ + --output "$ROLLOUTS_JSONL" \ + --num-repeats "$NUM_REPEATS" \ + --concurrency "$NUM_SAMPLES_IN_PARALLEL" + +gym eval profile \ + --inputs "$MATERIALIZED_JSONL" \ + --rollouts "$ROLLOUTS_JSONL" ``` If rows already contain `agent_ref`, leave `AGENT_NAME` empty. Passing `+agent_name` supplies a default for rows without one. ## Partial Rollouts -By default, `ng_reward_profile` expects every materialized input row to have a matching rollout row. If collection stopped early, profile the completed rollouts with: +By default, `gym eval profile` expects every materialized input row to have a matching rollout row. If collection stopped early, profile the completed rollouts with: ```bash -ng_reward_profile \ - ++materialized_inputs_jsonl_fpath="$MATERIALIZED_JSONL" \ - ++rollouts_jsonl_fpath="$ROLLOUTS_JSONL" \ +gym eval profile \ + --inputs "$MATERIALIZED_JSONL" \ + --rollouts "$ROLLOUTS_JSONL" \ ++allow_partial_rollouts=True ``` diff --git a/.codex/skills/nemo-gym-debugging/references/error-profiles.md b/.codex/skills/nemo-gym-debugging/references/error-profiles.md index 666b9eb9d4..7c9d540f65 100644 --- a/.codex/skills/nemo-gym-debugging/references/error-profiles.md +++ b/.codex/skills/nemo-gym-debugging/references/error-profiles.md @@ -113,7 +113,7 @@ Next actions: Symptoms: -- `ng_run` or `ng_collect_rollouts` fails before launching servers +- `gym env run` or `gym eval run --no-serve` fails before launching servers - errors mention missing config keys, unknown overrides, or interpolation failures - wrong agent/resource server starts despite expected data @@ -134,7 +134,7 @@ Evidence to collect: Next actions: -- reduce to a minimal `ng_run "+config_paths=[...]"` command +- reduce to a minimal `gym env run --config ...` command - add overrides back one at a time - trust target checkout code over copied command templates diff --git a/.codex/skills/nemo-gym-debugging/references/request-boundary-visibility.md b/.codex/skills/nemo-gym-debugging/references/request-boundary-visibility.md index c992750f40..0b83647324 100644 --- a/.codex/skills/nemo-gym-debugging/references/request-boundary-visibility.md +++ b/.codex/skills/nemo-gym-debugging/references/request-boundary-visibility.md @@ -9,7 +9,7 @@ This is an escalation ladder. Do not patch agents, resource servers, model adapt 1. Enable Gym's existing request debug flag: ```bash - ng_run '+config_paths=[...]' ++global_aiohttp_client_request_debug=True + gym env run --config ... ++global_aiohttp_client_request_debug=True ``` 2. Re-run the smallest failing case and capture stdout/stderr. diff --git a/.codex/skills/nemo-gym-reward-profiling/SKILL.md b/.codex/skills/nemo-gym-reward-profiling/SKILL.md index 89e99f1f5e..ab7067698d 100644 --- a/.codex/skills/nemo-gym-reward-profiling/SKILL.md +++ b/.codex/skills/nemo-gym-reward-profiling/SKILL.md @@ -2,7 +2,7 @@ name: nemo-gym-reward-profiling description: >- Use to help users get started with Nemo Gym reward profiling. Covers the basic - ng_run, ng_collect_rollouts, and ng_reward_profile workflow, repeated rollouts, + gym env run, gym eval run, and gym eval profile workflow, repeated rollouts, materialized inputs, rollout JSONL artifacts, task and rollout identity, output inspection, partial profiling, and rollout_infos. For failed jobs, prefer nemo-gym-debugging. @@ -14,16 +14,16 @@ description: >- Use this skill when the user wants to run, understand, or lightly modify Nemo Gym reward profiling. Keep the answer oriented around the normal workflow: -`ng_run` starts model/resource servers, `ng_collect_rollouts` writes rollout artifacts, and `ng_reward_profile` generates profiling output from those artifacts. +`gym env run` starts model/resource servers, `gym eval run --no-serve` writes rollout artifacts, and `gym eval profile` generates profiling output from those artifacts. -If the user is primarily debugging a failed job or stack trace, use `$nemo-gym-debugging` first. +If the user is primarily debugging a failed job or stack trace, use the `nemo-gym-debugging` skill first. ## Basic Workflow 1. Identify the environment config paths and input JSONL. -2. Start Gym servers with `ng_run`. -3. Collect rollouts with `ng_collect_rollouts`; this writes `rollouts.jsonl` and `*_materialized_inputs.jsonl`. -4. Run `ng_reward_profile` on the materialized inputs and rollout JSONL to generate `*_reward_profiling.jsonl`. +2. Start Gym servers with `gym env run`. +3. Collect rollouts with `gym eval run --no-serve`; this writes `rollouts.jsonl` and `*_materialized_inputs.jsonl`. +4. Run `gym eval profile` on the materialized inputs and rollout JSONL to generate `*_reward_profiling.jsonl`. 5. Inspect line counts and profile rows. Repeated rollouts are the main profiling lever. `num_repeats=1` is valid, but per-task averages and variance are only meaningful with multiple rollouts per task. @@ -48,6 +48,6 @@ Load references only when the user needs that detail: ## Practical Defaults -- Treat `ng_reward_profile` as the reward profiling step; rollout collection does not write reward profile files. +- Treat `gym eval profile` as the reward profiling step; rollout collection does not write reward profile files. - Run strict profiling by default. If rollout collection stopped early, use `++allow_partial_rollouts=True` to profile completed rollouts and drop original input rows with no completed rollout. - Trust the target checkout's CLI help and `nemo_gym/reward_profile.py` over memory if flags differ. diff --git a/.codex/skills/nemo-gym-reward-profiling/references/output-format.md b/.codex/skills/nemo-gym-reward-profiling/references/output-format.md index 71ab7b572d..23ab8dbfd3 100644 --- a/.codex/skills/nemo-gym-reward-profiling/references/output-format.md +++ b/.codex/skills/nemo-gym-reward-profiling/references/output-format.md @@ -6,7 +6,7 @@ Reward profiling is built around joining materialized inputs with rollout result - `*_materialized_inputs.jsonl`: expanded inputs after repeat expansion. Each row should have `_ng_task_index` and `_ng_rollout_index`. - `rollouts.jsonl`: completed rollout results. Each row should have matching `_ng_task_index` and `_ng_rollout_index`. -- `*_reward_profiling.jsonl`: task-level summaries produced by `ng_reward_profile`. +- `*_reward_profiling.jsonl`: task-level summaries produced by `gym eval profile`. - `*_agent_metrics.json`: agent/global aggregate metrics. ## Task Profile Rows @@ -36,7 +36,7 @@ Full model responses stay in `rollouts.jsonl`. Join back to full rows with `(_ng ## Partial Profiles -Strict profiling is the default. If materialized inputs and rollout results do not have the same rollout keys, `ng_reward_profile` fails and suggests: +Strict profiling is the default. If materialized inputs and rollout results do not have the same rollout keys, `gym eval profile` fails and suggests: ```bash ++allow_partial_rollouts=True diff --git a/.codex/skills/nemo-gym-reward-profiling/references/quick-start.md b/.codex/skills/nemo-gym-reward-profiling/references/quick-start.md index 0e2c80838b..9626f30605 100644 --- a/.codex/skills/nemo-gym-reward-profiling/references/quick-start.md +++ b/.codex/skills/nemo-gym-reward-profiling/references/quick-start.md @@ -5,8 +5,6 @@ Substitute environment-specific config paths, input data, model endpoint, and ou ## Minimal Flow ```bash -CONFIG_PATHS="your_model_config_paths,your_env_config_paths" - POLICY_MODEL_NAME="your_policy_model_name" POLICY_BASE_URL="your_policy_base_url" POLICY_ENDPOINT_KEY="your_policy_endpoint_key" @@ -19,41 +17,43 @@ AGENT_NAME="your_agent_name" NUM_REPEATS=2 NUM_SAMPLES_IN_PARALLEL=8 -ng_run "+config_paths=[$CONFIG_PATHS]" \ - +policy_model_name="$POLICY_MODEL_NAME" \ - +policy_base_url="$POLICY_BASE_URL" \ - +policy_api_key="$POLICY_ENDPOINT_KEY" & +gym env run \ + --config your_model_config_paths \ + --config your_env_config_paths \ + --model "$POLICY_MODEL_NAME" \ + --model-url "$POLICY_BASE_URL" \ + --model-api-key "$POLICY_ENDPOINT_KEY" & NG_RUN_PID=$! trap 'kill "$NG_RUN_PID" 2>/dev/null || true' EXIT ./scripts/wait_for_servers.sh "$NG_RUN_PID" agent_args=() if [[ -n "$AGENT_NAME" ]]; then - agent_args=(+agent_name="$AGENT_NAME") + agent_args=(--agent "$AGENT_NAME") fi -ng_collect_rollouts \ +gym eval run --no-serve \ "${agent_args[@]}" \ - +input_jsonl_fpath="$DATA_JSONL" \ - +output_jsonl_fpath="$ROLLOUTS_JSONL" \ - +num_repeats="$NUM_REPEATS" \ - +num_samples_in_parallel="$NUM_SAMPLES_IN_PARALLEL" - -ng_reward_profile \ - ++materialized_inputs_jsonl_fpath="$MATERIALIZED_JSONL" \ - ++rollouts_jsonl_fpath="$ROLLOUTS_JSONL" + --input "$DATA_JSONL" \ + --output "$ROLLOUTS_JSONL" \ + --num-repeats "$NUM_REPEATS" \ + --concurrency "$NUM_SAMPLES_IN_PARALLEL" + +gym eval profile \ + --inputs "$MATERIALIZED_JSONL" \ + --rollouts "$ROLLOUTS_JSONL" ``` If rows already contain `agent_ref`, leave `AGENT_NAME` empty. Passing `+agent_name` supplies a default for rows without one. ## Partial Rollouts -By default, `ng_reward_profile` expects every materialized input row to have a matching rollout row. If collection stopped early, profile the completed rollouts with: +By default, `gym eval profile` expects every materialized input row to have a matching rollout row. If collection stopped early, profile the completed rollouts with: ```bash -ng_reward_profile \ - ++materialized_inputs_jsonl_fpath="$MATERIALIZED_JSONL" \ - ++rollouts_jsonl_fpath="$ROLLOUTS_JSONL" \ +gym eval profile \ + --inputs "$MATERIALIZED_JSONL" \ + --rollouts "$ROLLOUTS_JSONL" \ ++allow_partial_rollouts=True ``` diff --git a/CLAUDE.md b/CLAUDE.md index d14cab2cf4..e7df93a3a7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,14 +82,16 @@ uv venv && uv sync --extra dev --group docs pre-commit install # Run servers -ng_run "+config_paths=[resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --resource-server example_single_tool_call \ + --model-type vllm_model # Run tests for a specific server (creates .venv per server, installs deps, runs pytest) # First run is slow. Use skip_venv_if_present config or place a .venv to skip venv creation. -ng_test +entrypoint=resources_servers/example_single_tool_call +gym env test --resource-server example_single_tool_call # Run all server tests -ng_test_all +gym env test # Run core library unit tests pytest tests/unit_tests/ -x @@ -105,13 +107,13 @@ ruff format . pre-commit run --all-files # Check server health -ng_status +gym env status -# Dev test (runs pytest directly in server dir, no venv isolation) -ng_dev_test +entrypoint=resources_servers/example_single_tool_call +# Dev test (runs the core unit tests with coverage: pytest --cov) +gym dev test # Dump merged config -ng_dump_config "+config_paths=[...]" +gym env resolve --config ... ``` ## Code Style @@ -143,4 +145,4 @@ git checkout -- resources_servers/other_server/ ## Cluster / HPC Gotchas - **Ray socket path length**: On systems with long working directory paths (e.g. Lustre mounts), Ray's AF_UNIX socket paths can exceed the 107-byte Linux limit. Fix: `RAY_TMPDIR=/tmp` before running tests or `ray.init()`. -- **`ng_test` venv isolation**: `ng_test` creates isolated venvs per resources server. `os.environ` changes in Python don't propagate — set env vars externally (e.g. `RAY_TMPDIR=/tmp ng_test ...`). +- **`gym env test` venv isolation**: `gym env test` creates isolated venvs per resources server. `os.environ` changes in Python don't propagate — set env vars externally (e.g. `RAY_TMPDIR=/tmp gym env test ...`). diff --git a/README.md b/README.md index ff13db7f3b..0f1bbddb41 100644 --- a/README.md +++ b/README.md @@ -100,10 +100,9 @@ Run your agent on a set of tasks and score the results. This example uses a simp NeMo Gym uses local servers to coordinate your model, agent, and task verification. Start them first: ```bash -environment_config="resources_servers/mcqa/configs/mcqa.yaml" -model_config="responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[${environment_config},${model_config}]" +gym env run \ + --resource-server mcqa \ + --model-type openai_model ``` You should see three server instances starting: @@ -121,12 +120,12 @@ In a new terminal, run your agent on a single task to verify everything works: ```bash source .venv/bin/activate -ng_collect_rollouts \ - +agent_name=mcqa_simple_agent \ - +input_jsonl_fpath=resources_servers/mcqa/data/example.jsonl \ - +output_jsonl_fpath=results/mcqa_rollouts.jsonl \ - +limit=5 \ - +num_repeats=1 +gym eval run --no-serve \ + --agent mcqa_simple_agent \ + --input resources_servers/mcqa/data/example.jsonl \ + --output results/mcqa_rollouts.jsonl \ + --limit 5 \ + --num-repeats 1 ``` You should see a progress bar followed by aggregate metrics: @@ -146,7 +145,7 @@ Rollouts: results/mcqa_rollouts.jsonl Aggregate metrics: results/mcqa_rollouts_aggregate_metrics.json ``` -For per-task pass rates, see the [`ng_reward_profile`](https://docs.nvidia.com/nemo/gym/main/reference/cli-commands) command. +For per-task pass rates, see the [`gym eval profile`](https://docs.nvidia.com/nemo/gym/main/reference/cli-commands) command. ### Next Steps diff --git a/benchmarks/aalcr/README.md b/benchmarks/aalcr/README.md index 52a0d823d0..996e24f4fa 100644 --- a/benchmarks/aalcr/README.md +++ b/benchmarks/aalcr/README.md @@ -1,15 +1,13 @@ # Prepare data ```bash -config_paths="benchmarks/aalcr/config.yaml" -ng_prepare_benchmark "+config_paths=[$config_paths]" +gym eval prepare --benchmark aalcr ``` # Run ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/aalcr/config.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ +gym eval run \ + --model-type vllm_model \ + --benchmark aalcr \ ++output_jsonl_fpath=results/benchmarks/aalcr.jsonl \ ++overwrite_metrics_conflicts=true \ ++split=benchmark \ diff --git a/benchmarks/aime24-x/README.md b/benchmarks/aime24-x/README.md index e59d3bfde3..776fc0e17b 100644 --- a/benchmarks/aime24-x/README.md +++ b/benchmarks/aime24-x/README.md @@ -21,7 +21,7 @@ This benchmark reuses `math_with_judge` in symbolic-only mode ## Data Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/aime24-x/config.yaml]" +gym eval prepare --benchmark aime24-x ``` That writes `benchmarks/aime24-x/data/aime24-x_benchmark.jsonl`. @@ -36,16 +36,21 @@ python benchmarks/aime24-x/prepare.py --prompt_language en ## Quickstart ```bash -ng_run "+config_paths=[benchmarks/aime24-x/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --benchmark aime24-x \ + --model-type vllm_model ``` Then in another shell: ```bash -ng_collect_rollouts \ - +agent_name=aime24-x_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/aime24-x/data/aime24-x_benchmark.jsonl \ - +output_jsonl_fpath=results/aime24-x/rollouts.jsonl \ - +num_repeats=32 +num_repeats_add_seed=true \ - "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 65536}" +gym eval run --no-serve \ + --agent aime24-x_math_with_judge_simple_agent \ + --input benchmarks/aime24-x/data/aime24-x_benchmark.jsonl \ + --output results/aime24-x/rollouts.jsonl \ + --num-repeats 32 \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 65536 \ + +num_repeats_add_seed=true ``` diff --git a/benchmarks/aime25-x/README.md b/benchmarks/aime25-x/README.md index 1ac1453ff1..6ae97da3cd 100644 --- a/benchmarks/aime25-x/README.md +++ b/benchmarks/aime25-x/README.md @@ -21,7 +21,7 @@ This benchmark reuses `math_with_judge` in symbolic-only mode ## Data Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/aime25-x/config.yaml]" +gym eval prepare --benchmark aime25-x ``` That writes `benchmarks/aime25-x/data/aime25-x_benchmark.jsonl`. @@ -36,17 +36,22 @@ python benchmarks/aime25-x/prepare.py --prompt_language en ## Quickstart ```bash -ng_run "+config_paths=[benchmarks/aime25-x/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --benchmark aime25-x \ + --model-type vllm_model ``` Then in another shell: ```bash -ng_collect_rollouts \ - +agent_name=aime25-x_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/aime25-x/data/aime25-x_benchmark.jsonl \ - +output_jsonl_fpath=results/aime25-x/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml \ - +num_repeats=32 +num_repeats_add_seed=true \ - "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 65536}" +gym eval run --no-serve \ + --agent aime25-x_math_with_judge_simple_agent \ + --input benchmarks/aime25-x/data/aime25-x_benchmark.jsonl \ + --output results/aime25-x/rollouts.jsonl \ + --num-repeats 32 \ + --prompt-config benchmarks/prompts/generic/default.yaml \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 65536 \ + +num_repeats_add_seed=true ``` diff --git a/benchmarks/aime26/README.md b/benchmarks/aime26/README.md index a7d0c401d6..279c3c4f1a 100644 --- a/benchmarks/aime26/README.md +++ b/benchmarks/aime26/README.md @@ -5,22 +5,23 @@ AIME 2026 (American Invitational Mathematics Examination) — 30 competition mat ## Prepare data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/aime26/config.yaml]" +gym eval prepare --benchmark aime26 ``` ## Run servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/aime26/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark aime26 ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=aime26_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/aime26/data/aime26_benchmark.jsonl \ - +output_jsonl_fpath=results/aime26_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent aime26_math_with_judge_simple_agent \ + --input benchmarks/aime26/data/aime26_benchmark.jsonl \ + --output results/aime26_rollouts.jsonl \ + --num-repeats 4 ``` diff --git a/benchmarks/answer-judge/README.md b/benchmarks/answer-judge/README.md index 3313ae000c..ffd6fa066b 100644 --- a/benchmarks/answer-judge/README.md +++ b/benchmarks/answer-judge/README.md @@ -13,17 +13,17 @@ here is the same deterministic `Judgement: Yes/No` parsing used by Skills' ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/answer-judge/config.yaml]" +gym eval prepare --benchmark answer-judge # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/answer-judge/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark answer-judge # Collecting rollouts -ng_collect_rollouts \ - +agent_name=answer_judge_math_proof_judgement_simple_agent \ - +input_jsonl_fpath=benchmarks/answer-judge/data/answer-judge_benchmark.jsonl \ - +output_jsonl_fpath=results/answer-judge/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/judge/math.yaml +gym eval run --no-serve \ + --agent answer_judge_math_proof_judgement_simple_agent \ + --input benchmarks/answer-judge/data/answer-judge_benchmark.jsonl \ + --output results/answer-judge/rollouts.jsonl \ + --prompt-config benchmarks/prompts/judge/math.yaml ``` diff --git a/benchmarks/apex_shortlist/README.md b/benchmarks/apex_shortlist/README.md index 2ced13a9f8..526f69c7fa 100644 --- a/benchmarks/apex_shortlist/README.md +++ b/benchmarks/apex_shortlist/README.md @@ -26,7 +26,7 @@ Solve the following math problem. Make sure to put the answer (and only answer) ## Data preparation ```bash -ng_prepare_benchmark '+config_paths=[benchmarks/apex_shortlist/config.yaml]' +gym eval prepare --benchmark apex_shortlist ``` Writes `data/apex_shortlist_benchmark.jsonl` with one row per problem: @@ -35,17 +35,17 @@ Writes `data/apex_shortlist_benchmark.jsonl` with one row per problem: ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/apex_shortlist/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark apex_shortlist ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=apex_shortlist_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/apex_shortlist/data/apex_shortlist_benchmark.jsonl \ - +output_jsonl_fpath=results/apex_shortlist_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent apex_shortlist_math_with_judge_simple_agent \ + --input benchmarks/apex_shortlist/data/apex_shortlist_benchmark.jsonl \ + --output results/apex_shortlist_rollouts.jsonl \ + --num-repeats 4 ``` diff --git a/benchmarks/arena_hard/README.md b/benchmarks/arena_hard/README.md index 55e75997b5..ab334cd230 100644 --- a/benchmarks/arena_hard/README.md +++ b/benchmarks/arena_hard/README.md @@ -15,7 +15,7 @@ for the judging protocol and metric details. ## Data Runtime download only — benchmark JSONL is not committed. Run -[`prepare.py`](prepare.py) (or `ng_prepare_benchmark`) to populate +[`prepare.py`](prepare.py) (or `gym eval prepare`) to populate `data/arena_hard_benchmark.jsonl`. The prepare script fetches questions and the baseline directly from the arena-hard-auto GitHub repo, joins by `uid`, and emits one row per question with `question`, @@ -28,20 +28,20 @@ to pick the standard judge prompt. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/arena_hard/config.yaml]" +gym eval prepare --benchmark arena_hard # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/arena_hard/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark arena_hard # Collecting rollouts -ng_collect_rollouts \ - +agent_name=arena_hard_arena_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/arena_hard/data/arena_hard_benchmark.jsonl \ - +output_jsonl_fpath=results/arena_hard_rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent arena_hard_arena_judge_simple_agent \ + --input benchmarks/arena_hard/data/arena_hard_benchmark.jsonl \ + --output results/arena_hard_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/prompts/generic/default.yaml ``` ## Metrics diff --git a/benchmarks/arena_hard_v2/README.md b/benchmarks/arena_hard_v2/README.md index e88a228525..31a4d551fa 100644 --- a/benchmarks/arena_hard_v2/README.md +++ b/benchmarks/arena_hard_v2/README.md @@ -21,7 +21,7 @@ for the judging protocol and metric details. ## Data Runtime download only — benchmark JSONL is not committed. Run -[`prepare.py`](prepare.py) (or `ng_prepare_benchmark`) to populate +[`prepare.py`](prepare.py) (or `gym eval prepare`) to populate `data/arena_hard_v2_benchmark.jsonl`. The prepare script fetches questions and both baselines directly from the arena-hard-auto GitHub repo, joins by `uid`, and emits one row per question with `question`, @@ -31,19 +31,19 @@ repo, joins by `uid`, and emits one row per question with `question`, ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/arena_hard_v2/config.yaml]" +gym eval prepare --benchmark arena_hard_v2 # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/arena_hard_v2/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark arena_hard_v2 # Collecting rollouts -ng_collect_rollouts \ - +agent_name=arena_hard_v2_arena_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/arena_hard_v2/data/arena_hard_v2_benchmark.jsonl \ - +output_jsonl_fpath=results/arena_hard_v2_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent arena_hard_v2_arena_judge_simple_agent \ + --input benchmarks/arena_hard_v2/data/arena_hard_v2_benchmark.jsonl \ + --output results/arena_hard_v2_rollouts.jsonl \ + --num-repeats 4 ``` ## Metrics diff --git a/benchmarks/asr_leaderboard/README.md b/benchmarks/asr_leaderboard/README.md index 7326651f22..0d1883fa0c 100644 --- a/benchmarks/asr_leaderboard/README.md +++ b/benchmarks/asr_leaderboard/README.md @@ -23,7 +23,7 @@ System + user templates live in [`prompts/default.yaml`](prompts/default.yaml). ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/asr_leaderboard/config.yaml]" +gym eval prepare --benchmark asr_leaderboard ``` Downloads the 8 ESB subsets (~tens of GB of FLAC) and writes @@ -32,18 +32,18 @@ Downloads the 8 ESB subsets (~tens of GB of FLAC) and writes ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/asr_leaderboard/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark asr_leaderboard ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=asr_leaderboard_asr_with_pc_simple_agent \ - +output_jsonl_fpath=results/asr_leaderboard_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent asr_leaderboard_asr_with_pc_simple_agent \ + --output results/asr_leaderboard_rollouts.jsonl \ + --num-repeats 1 ``` ## Verification diff --git a/benchmarks/bigcodebench/README.md b/benchmarks/bigcodebench/README.md index 0722b2eaa9..16360c05bf 100644 --- a/benchmarks/bigcodebench/README.md +++ b/benchmarks/bigcodebench/README.md @@ -13,27 +13,27 @@ the `full` split (~1140 problems) is `bigcode/bigcodebench@v0.1.4`. ```bash # Prepare benchmark data (hard split, ~148 problems) -ng_prepare_benchmark "+config_paths=[benchmarks/bigcodebench/config.yaml]" +gym eval prepare --benchmark bigcodebench # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/bigcodebench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark bigcodebench # Collecting rollouts (5-row smoke test against the baked example set) -ng_collect_rollouts \ - +agent_name=bigcodebench_benchmark_simple_agent \ - +input_jsonl_fpath=resources_servers/bigcodebench/data/example.jsonl \ - +output_jsonl_fpath=results/bigcodebench_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent bigcodebench_benchmark_simple_agent \ + --input resources_servers/bigcodebench/data/example.jsonl \ + --output results/bigcodebench_rollouts.jsonl \ + --num-repeats 1 ``` -The benchmark JSONL written by ``ng_prepare_benchmark`` is unbaked +The benchmark JSONL written by ``gym eval prepare`` is unbaked (rows have ``question`` + ``verifier_metadata``; the prompt template is -applied by the agent at ``/run`` time). Standalone ``ng_collect_rollouts`` +applied by the agent at ``/run`` time). Standalone ``gym eval run --no-serve`` expects pre-baked ``responses_create_params.input``, so for full-dataset runs use the production orchestrator (``nemo_gym_rollouts`` from -NeMo-Skills) rather than ``ng_collect_rollouts`` directly. +NeMo-Skills) rather than ``gym eval run --no-serve`` directly. `prepare.py` exposes a `--split` flag (`hard` or `full`); the config defaults to `hard` to match the recipe's parity-comparison run. diff --git a/benchmarks/birdbench/README.md b/benchmarks/birdbench/README.md index ba7708c352..f6e198e003 100644 --- a/benchmarks/birdbench/README.md +++ b/benchmarks/birdbench/README.md @@ -10,7 +10,7 @@ Execution-based text-to-SQL on BIRD dev, bound to the `bird_sql` resource server ## Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/birdbench/config.yaml]" +gym eval prepare --benchmark birdbench ``` This downloads the BIRD `dev.zip` (≈1.4 GB) via @@ -22,9 +22,9 @@ and writes `data/birdbench_benchmark.jsonl`. Each row has ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/birdbench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark birdbench ``` Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in @@ -33,11 +33,11 @@ Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=birdbench_bird_sql_simple_agent \ - +input_jsonl_fpath=benchmarks/birdbench/data/birdbench_benchmark.jsonl \ - +output_jsonl_fpath=results/birdbench_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent birdbench_bird_sql_simple_agent \ + --input benchmarks/birdbench/data/birdbench_benchmark.jsonl \ + --output results/birdbench_rollouts.jsonl \ + --num-repeats 4 ``` For a 5-example smoke test against the resource server's `example.jsonl`, diff --git a/benchmarks/browsecomp/README.md b/benchmarks/browsecomp/README.md index 7d7d2ef9dd..4d28ef61c1 100644 --- a/benchmarks/browsecomp/README.md +++ b/benchmarks/browsecomp/README.md @@ -15,23 +15,21 @@ Qwen3-235B-A22B-Instruct-2507-FP8: 2. Prepare the benchmark dataset ```bash -config_paths="benchmarks/browsecomp/config.yaml" -ng_prepare_benchmark "+config_paths=[$config_paths]" +gym eval prepare --benchmark browsecomp ``` 3. Run the benchmark against a VLLMModel ```bash WANDB_PROJECT= EXPERIMENT_NAME= -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/browsecomp/config.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ +gym eval run \ + --model-type vllm_model \ + --benchmark browsecomp \ + --output results/$EXPERIMENT_NAME.jsonl \ + --split benchmark \ + --model-url ??? \ + --model-api-key ??? \ + --model ??? \ +wandb_project=$WANDB_PROJECT \ - +wandb_name=$EXPERIMENT_NAME \ - ++output_jsonl_fpath=results/$EXPERIMENT_NAME.jsonl \ - ++split=benchmark \ - ++policy_base_url=??? \ - ++policy_api_key=??? \ - ++policy_model_name=??? + +wandb_name=$EXPERIMENT_NAME ``` diff --git a/benchmarks/finance_sec_search/README.md b/benchmarks/finance_sec_search/README.md index 5a43b6c9fb..deb3a4c675 100644 --- a/benchmarks/finance_sec_search/README.md +++ b/benchmarks/finance_sec_search/README.md @@ -27,13 +27,13 @@ server's `/prompt_templates`. Without web search: ```bash -ng_prepare_benchmark '+config_paths=[benchmarks/finance_sec_search/config_no_web_search.yaml]' +gym eval prepare --benchmark finance_sec_search/config_no_web_search ``` With web search (requires `tavily_api_key` in `env.yaml`): ```bash -ng_prepare_benchmark '+config_paths=[benchmarks/finance_sec_search/config_web_search.yaml]' +gym eval prepare --benchmark finance_sec_search/config_web_search ``` Downloads `public.csv` from the Vals AI GitHub repo and writes benchmark @@ -47,16 +47,16 @@ JSONL to `data/`. ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/finance_sec_search/config_no_web_search.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark finance_sec_search/config_no_web_search ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=finance_sec_search_benchmark_agent \ - +input_jsonl_fpath=benchmarks/finance_sec_search/data/finance_sec_search_benchmark.jsonl \ - +output_jsonl_fpath=results/finance_sec_search_rollouts.jsonl +gym eval run --no-serve \ + --agent finance_sec_search_benchmark_agent \ + --input benchmarks/finance_sec_search/data/finance_sec_search_benchmark.jsonl \ + --output results/finance_sec_search_rollouts.jsonl ``` diff --git a/benchmarks/flores200/README.md b/benchmarks/flores200/README.md index 8f770cbedd..9b272195ef 100644 --- a/benchmarks/flores200/README.md +++ b/benchmarks/flores200/README.md @@ -18,7 +18,7 @@ the Ray GPU-scheduled COMET path. ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/flores200/config.yaml]" +gym eval prepare --benchmark flores200 ``` Writes `data/flores200_devtest_benchmark.jsonl` and pre-fetches the @@ -41,21 +41,21 @@ only advertised on multi-node SLURM deployments via NeMo-Skills' override and rely on corpus-BLEU only: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/flores200/config.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --model-type vllm_model \ + --benchmark flores200 \ "++flores200_wmt_translation_resources_server.resources_servers.wmt_translation.compute_comet=false" ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=flores200_wmt_translation_simple_agent \ - +prompt_config=benchmarks/flores200/prompts/default.yaml \ - +input_jsonl_fpath=benchmarks/flores200/data/flores200_devtest_benchmark.jsonl \ - +output_jsonl_fpath=results/flores200_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent flores200_wmt_translation_simple_agent \ + --input benchmarks/flores200/data/flores200_devtest_benchmark.jsonl \ + --output results/flores200_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/flores200/prompts/default.yaml ``` ## End-to-end on a SLURM cluster (via NeMo-Skills) @@ -76,7 +76,7 @@ ns run_cmd \ --cluster \ --container nemo-gym \ --expname flores200_prepare \ - --command 'ng_prepare_benchmark "+config_paths=[benchmarks/flores200/config.yaml]"' + --command 'gym eval prepare --benchmark flores200' ``` This populates `benchmarks/flores200/data/flores200_devtest_benchmark.jsonl` diff --git a/benchmarks/frontierscience_olympiad/README.md b/benchmarks/frontierscience_olympiad/README.md index 645a646e64..e7c631df5e 100644 --- a/benchmarks/frontierscience_olympiad/README.md +++ b/benchmarks/frontierscience_olympiad/README.md @@ -36,20 +36,20 @@ example, the original Skills configuration uses `o3-mini-2025-01-31` via ```bash # Prepare benchmark data (downloads from HuggingFace) -ng_prepare_benchmark "+config_paths=[benchmarks/frontierscience_olympiad/config.yaml]" +gym eval prepare --benchmark frontierscience_olympiad # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/frontierscience_olympiad/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark frontierscience_olympiad # Collecting rollouts (4 rollouts per task, matching Skills' default) -ng_collect_rollouts \ - +agent_name=frontierscience_olympiad_frontierscience_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/frontierscience_olympiad/data/frontierscience_olympiad_benchmark.jsonl \ - +prompt_config=benchmarks/frontierscience_olympiad/prompts/default.yaml \ - +output_jsonl_fpath=results/frontierscience_olympiad_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent frontierscience_olympiad_frontierscience_judge_simple_agent \ + --input benchmarks/frontierscience_olympiad/data/frontierscience_olympiad_benchmark.jsonl \ + --prompt-config benchmarks/frontierscience_olympiad/prompts/default.yaml \ + --output results/frontierscience_olympiad_rollouts.jsonl \ + --num-repeats 4 ``` For Nemotron-3-Nano and other reasoning models, start vLLM with diff --git a/benchmarks/gdpval/README.md b/benchmarks/gdpval/README.md index b2e50937f8..e1029ebbaa 100644 --- a/benchmarks/gdpval/README.md +++ b/benchmarks/gdpval/README.md @@ -11,7 +11,7 @@ Downloads `openai/gdpval` from HuggingFace and writes `data/gdpval_benchmark.jsonl`: ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/gdpval/config.yaml]" +gym eval prepare --benchmark gdpval ``` ## Run rubric mode (default) @@ -19,15 +19,14 @@ ng_prepare_benchmark "+config_paths=[benchmarks/gdpval/config.yaml]" Each deliverable is scored 0–1 against the task rubric. ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/gdpval/config.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++output_jsonl_fpath=results/gdpval_rubric.jsonl \ - ++split=benchmark \ - ++policy_base_url= \ - ++policy_api_key= \ - ++policy_model_name= +gym eval run \ + --model-type vllm_model \ + --benchmark gdpval \ + --output results/gdpval_rubric.jsonl \ + --split benchmark \ + --model-url \ + --model-api-key \ + --model ``` Required environment variables for the judge: @@ -45,10 +44,11 @@ same `task_id`; aggregate metrics include ELO relative to a configurable anchor (default 1000). ```bash -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++output_jsonl_fpath=results/gdpval_compare.jsonl \ - ++split=benchmark \ +gym eval run \ + --model-type vllm_model \ + --benchmark gdpval \ + --output results/gdpval_compare.jsonl \ + --split benchmark \ ++gdpval_resources_server.resources_servers.gdpval.reward_mode=comparison \ ++gdpval_resources_server.resources_servers.gdpval.reference_deliverables_dir=/path/to/reference/output ``` @@ -59,7 +59,7 @@ the deliverable files (the same layout the Stirrup agent persists). ## Aggregate metrics -After `ng_e2e_collect_rollouts` returns, the resources server's +After `gym eval run` returns, the resources server's `/aggregate_metrics` endpoint emits headline scores in `results/_metrics.json`: diff --git a/benchmarks/global-piqa/README.md b/benchmarks/global-piqa/README.md index 73516ff902..b3048b92a6 100644 --- a/benchmarks/global-piqa/README.md +++ b/benchmarks/global-piqa/README.md @@ -16,17 +16,17 @@ server. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/global-piqa/config.yaml]" +gym eval prepare --benchmark global-piqa # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/global-piqa/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark global-piqa # Collecting rollouts -ng_collect_rollouts \ - +agent_name=global_piqa_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/global-piqa/data/global-piqa_benchmark.jsonl \ - +output_jsonl_fpath=results/global-piqa/rollouts.jsonl \ - +prompt_config=benchmarks/global-piqa/prompts/default.yaml +gym eval run --no-serve \ + --agent global_piqa_mcqa_simple_agent \ + --input benchmarks/global-piqa/data/global-piqa_benchmark.jsonl \ + --output results/global-piqa/rollouts.jsonl \ + --prompt-config benchmarks/global-piqa/prompts/default.yaml ``` diff --git a/benchmarks/gpqa-x/README.md b/benchmarks/gpqa-x/README.md index f50c9a8a69..6043283d44 100644 --- a/benchmarks/gpqa-x/README.md +++ b/benchmarks/gpqa-x/README.md @@ -22,7 +22,7 @@ This benchmark reuses the `mcqa` resource server, which matches Skills' ## Data Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/gpqa-x/config.yaml]" +gym eval prepare --benchmark gpqa-x ``` That writes `benchmarks/gpqa-x/data/gpqa-x_benchmark.jsonl`. @@ -37,17 +37,22 @@ python benchmarks/gpqa-x/prepare.py --prompt_language en ## Quickstart ```bash -ng_run "+config_paths=[benchmarks/gpqa-x/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --benchmark gpqa-x \ + --model-type vllm_model ``` Then in another shell: ```bash -ng_collect_rollouts \ - +agent_name=gpqa-x_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/gpqa-x/data/gpqa-x_benchmark.jsonl \ - +output_jsonl_fpath=results/gpqa-x/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml \ - +num_repeats=8 +num_repeats_add_seed=true \ - "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 32768}" +gym eval run --no-serve \ + --agent gpqa-x_mcqa_simple_agent \ + --input benchmarks/gpqa-x/data/gpqa-x_benchmark.jsonl \ + --output results/gpqa-x/rollouts.jsonl \ + --prompt-config benchmarks/prompts/generic/default.yaml \ + --num-repeats 8 \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 32768 \ + +num_repeats_add_seed=true ``` diff --git a/benchmarks/gpqa/README.md b/benchmarks/gpqa/README.md index 1801de0c52..679e9efb06 100644 --- a/benchmarks/gpqa/README.md +++ b/benchmarks/gpqa/README.md @@ -13,13 +13,16 @@ This benchmark uses the `mcqa` resource server with the `mcqa_simple_agent`. ```bash # Prepare data -ng_prepare_benchmark "+config_paths=[benchmarks/gpqa/config.yaml]" +gym eval prepare --benchmark gpqa # Start servers -ng_run "+config_paths=[benchmarks/gpqa/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --benchmark gpqa \ + --model-type vllm_model # Collect rollouts -ng_collect_rollouts \ - "+config_paths=[benchmarks/gpqa/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +output_jsonl_fpath=results/gpqa.jsonl +gym eval run --no-serve \ + --benchmark gpqa \ + --model-type vllm_model \ + --output results/gpqa.jsonl ``` diff --git a/benchmarks/graphwalks/README.md b/benchmarks/graphwalks/README.md index 526a6e9152..e486653735 100644 --- a/benchmarks/graphwalks/README.md +++ b/benchmarks/graphwalks/README.md @@ -28,10 +28,10 @@ The N3 1M variant requires HF auth for the gated NVIDIA repo ```bash # Default (o200k_base, no filter) -ng_prepare_benchmark "+config_paths=[benchmarks/graphwalks/config.yaml]" +gym eval prepare --benchmark graphwalks # N3 1M variant -ng_prepare_benchmark "+config_paths=[benchmarks/graphwalks/config_n3_1m.yaml]" +gym eval prepare --benchmark graphwalks/config_n3_1m ``` For one-off custom builds (different tokenizer / cap / output path), @@ -47,25 +47,27 @@ python benchmarks/graphwalks/prepare.py \ ## Start environment ```bash -ng_run "+config_paths=[benchmarks/graphwalks/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --benchmark graphwalks \ + --model-type vllm_model ``` ## Collect rollouts ```bash # Default variant -ng_collect_rollouts \ - +agent_name=graphwalks_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/graphwalks/data/graphwalks_benchmark.jsonl \ - +output_jsonl_fpath=results/graphwalks_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent graphwalks_benchmark_simple_agent \ + --input benchmarks/graphwalks/data/graphwalks_benchmark.jsonl \ + --output results/graphwalks_rollouts.jsonl \ + --num-repeats 4 # N3 1M variant -ng_collect_rollouts \ - +agent_name=graphwalks_n3_1m_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/graphwalks/data/graphwalks_n3_1m_benchmark.jsonl \ - +output_jsonl_fpath=results/graphwalks_n3_1m_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent graphwalks_n3_1m_benchmark_simple_agent \ + --input benchmarks/graphwalks/data/graphwalks_n3_1m_benchmark.jsonl \ + --output results/graphwalks_n3_1m_rollouts.jsonl \ + --num-repeats 4 ``` ## Metrics diff --git a/benchmarks/gsm8k/README.md b/benchmarks/gsm8k/README.md index d9a425048d..ef4b9d917c 100644 --- a/benchmarks/gsm8k/README.md +++ b/benchmarks/gsm8k/README.md @@ -13,17 +13,17 @@ the expected answer is integer-valued), then renames `problem` -> ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/gsm8k/config.yaml]" +gym eval prepare --benchmark gsm8k # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/gsm8k/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark gsm8k # Collecting rollouts -ng_collect_rollouts \ - +agent_name=gsm8k_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/gsm8k/data/gsm8k_benchmark.jsonl \ - +output_jsonl_fpath=results/gsm8k_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent gsm8k_math_with_judge_simple_agent \ + --input benchmarks/gsm8k/data/gsm8k_benchmark.jsonl \ + --output results/gsm8k_rollouts.jsonl \ + --num-repeats 4 ``` diff --git a/benchmarks/hendrycks_math/README.md b/benchmarks/hendrycks_math/README.md index 1b51db38f5..d10253c03a 100644 --- a/benchmarks/hendrycks_math/README.md +++ b/benchmarks/hendrycks_math/README.md @@ -12,17 +12,17 @@ applies Skills' renames (`answer` -> `expected_answer`, `question` -> ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/hendrycks_math/config.yaml]" +gym eval prepare --benchmark hendrycks_math # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/hendrycks_math/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark hendrycks_math # Collecting rollouts -ng_collect_rollouts \ - +agent_name=hendrycks_math_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/hendrycks_math/data/hendrycks_math_benchmark.jsonl \ - +output_jsonl_fpath=results/hendrycks_math_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent hendrycks_math_math_with_judge_simple_agent \ + --input benchmarks/hendrycks_math/data/hendrycks_math_benchmark.jsonl \ + --output results/hendrycks_math_rollouts.jsonl \ + --num-repeats 4 ``` diff --git a/benchmarks/hle/README.md b/benchmarks/hle/README.md index 61d83fcb28..b08d6486ef 100644 --- a/benchmarks/hle/README.md +++ b/benchmarks/hle/README.md @@ -25,7 +25,7 @@ huggingface-cli login ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/hle/config.yaml]" +gym eval prepare --benchmark hle ``` Downloads `cais/hle`, filters to text-only questions, and writes @@ -34,9 +34,9 @@ Downloads `cais/hle`, filters to text-only questions, and writes ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/hle/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark hle ``` Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in @@ -45,13 +45,13 @@ Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=hle_equivalence_llm_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/hle/data/hle_benchmark.jsonl \ - +output_jsonl_fpath=results/hle_rollouts.jsonl \ - +prompt_config=benchmarks/hle/prompts/default.yaml \ - +num_repeats=1 \ - "++responses_create_params={temperature: 0.0}" +gym eval run --no-serve \ + --agent hle_equivalence_llm_judge_simple_agent \ + --input benchmarks/hle/data/hle_benchmark.jsonl \ + --output results/hle_rollouts.jsonl \ + --prompt-config benchmarks/hle/prompts/default.yaml \ + --num-repeats 1 \ + --temperature 0.0 ``` Use `temperature: 0.0` to match the nemo-skills evaluation setup and ensure reproducible scores. diff --git a/benchmarks/hmmt_feb25/README.md b/benchmarks/hmmt_feb25/README.md index 067dbbabb1..c890c0e390 100644 --- a/benchmarks/hmmt_feb25/README.md +++ b/benchmarks/hmmt_feb25/README.md @@ -25,7 +25,7 @@ Solve the following math problem. Make sure to put the answer (and only answer) ## Data preparation ``` -ng_prepare_benchmark '+config_paths=[benchmarks/hmmt_feb25/config.yaml]' +gym eval prepare --benchmark hmmt_feb25 ``` Writes `data/hmmt_feb25_benchmark.jsonl` with one row per problem: @@ -37,7 +37,9 @@ Start the benchmark's servers (inherits `math_with_judge` in symbolic-only mode plus a vLLM model server — adjust the model config to match your deployment): ``` -ng_run "+config_paths=[benchmarks/hmmt_feb25/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --benchmark hmmt_feb25 \ + --model-type vllm_model ``` In a separate shell, collect rollouts against the full 30-problem set. `num_repeats` @@ -45,13 +47,16 @@ controls rollouts-per-task for pass@k; use 16 for parity-grade evaluation, or dr to 4 for a faster smoke pass: ``` -ng_collect_rollouts \ - +agent_name=hmmt_feb25_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/hmmt_feb25/data/hmmt_feb25_benchmark.jsonl \ - +output_jsonl_fpath=results/hmmt_feb25/rollouts.jsonl \ - +prompt_config=benchmarks/hmmt_feb25/prompts/default.yaml \ - +num_repeats=16 +num_repeats_add_seed=true \ - "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 65536}" +gym eval run --no-serve \ + --agent hmmt_feb25_math_with_judge_simple_agent \ + --input benchmarks/hmmt_feb25/data/hmmt_feb25_benchmark.jsonl \ + --output results/hmmt_feb25/rollouts.jsonl \ + --prompt-config benchmarks/hmmt_feb25/prompts/default.yaml \ + --num-repeats 16 \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 65536 \ + +num_repeats_add_seed=true ``` `num_repeats_add_seed=true` assigns a distinct vLLM `seed` to each rollout via diff --git a/benchmarks/hmmt_nov25/README.md b/benchmarks/hmmt_nov25/README.md index 36cddd227f..98a4ab4b85 100644 --- a/benchmarks/hmmt_nov25/README.md +++ b/benchmarks/hmmt_nov25/README.md @@ -35,20 +35,22 @@ on the same inputs. ```bash # Prepare benchmark data (downloads from HuggingFace) -ng_prepare_benchmark "+config_paths=[benchmarks/hmmt_nov25/config.yaml]" +gym eval prepare --benchmark hmmt_nov25 # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/hmmt_nov25/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark hmmt_nov25 # Collecting rollouts -ng_collect_rollouts \ - +agent_name=hmmt_nov25_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/hmmt_nov25/data/hmmt_nov25_benchmark.jsonl \ - +output_jsonl_fpath=results/hmmt_nov25_rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/math.yaml \ - +num_repeats=16 \ - +num_repeats_add_seed=true \ - "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 65536}" +gym eval run --no-serve \ + --agent hmmt_nov25_math_with_judge_simple_agent \ + --input benchmarks/hmmt_nov25/data/hmmt_nov25_benchmark.jsonl \ + --output results/hmmt_nov25_rollouts.jsonl \ + --prompt-config benchmarks/prompts/generic/math.yaml \ + --num-repeats 16 \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 65536 \ + +num_repeats_add_seed=true ``` diff --git a/benchmarks/hotpotqa_closedbook/README.md b/benchmarks/hotpotqa_closedbook/README.md index 2f145701a4..1c5bb2166b 100644 --- a/benchmarks/hotpotqa_closedbook/README.md +++ b/benchmarks/hotpotqa_closedbook/README.md @@ -19,24 +19,25 @@ validation split — the exact same source used by Skills' (`question`, `expected_answer`, `id`, `type`, `level`). ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/hotpotqa_closedbook/config.yaml]" +gym eval prepare --benchmark hotpotqa_closedbook ``` ## Run servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/hotpotqa_closedbook/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark hotpotqa_closedbook ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=hotpotqa_closedbook_simple_agent \ - +input_jsonl_fpath=benchmarks/hotpotqa_closedbook/data/hotpotqa_closedbook_benchmark.jsonl \ - +output_jsonl_fpath=results/hotpotqa_closedbook_rollouts.jsonl \ - +num_repeats=4 \ +gym eval run --no-serve \ + --agent hotpotqa_closedbook_simple_agent \ + --input benchmarks/hotpotqa_closedbook/data/hotpotqa_closedbook_benchmark.jsonl \ + --output results/hotpotqa_closedbook_rollouts.jsonl \ + --num-repeats 4 \ +num_repeats_add_seed=true ``` diff --git a/benchmarks/human_eval/README.md b/benchmarks/human_eval/README.md index ade0190a0b..32436d3d90 100644 --- a/benchmarks/human_eval/README.md +++ b/benchmarks/human_eval/README.md @@ -20,23 +20,23 @@ holds the dataset definition + prompt + prepare script. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/human_eval/config.yaml]" +gym eval prepare --benchmark human_eval # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/human_eval/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark human_eval # Collecting rollouts. The +prompt_config= override is required because # the prepared JSONL stores raw `question` rows (no `responses_create_params.input`); -# ng_collect_rollouts does not pick up the `prompt_config:` field on the dataset -# entry in config.yaml the way ng_run does. -ng_collect_rollouts \ - +agent_name=human_eval_evalplus_simple_agent \ - +input_jsonl_fpath=benchmarks/human_eval/data/human_eval_benchmark.jsonl \ - +prompt_config=benchmarks/prompts/generic/codegen.yaml \ - +output_jsonl_fpath=results/human_eval_rollouts.jsonl \ - +num_repeats=4 +# gym eval run --no-serve does not pick up the `prompt_config:` field on the dataset +# entry in config.yaml the way gym env run does. +gym eval run --no-serve \ + --agent human_eval_evalplus_simple_agent \ + --input benchmarks/human_eval/data/human_eval_benchmark.jsonl \ + --prompt-config benchmarks/prompts/generic/codegen.yaml \ + --output results/human_eval_rollouts.jsonl \ + --num-repeats 4 ``` Start vLLM with `--reasoning-parser ` (e.g. `deepseek_r1` for diff --git a/benchmarks/human_eval_infilling/README.md b/benchmarks/human_eval_infilling/README.md index b5a301e830..76aa13f800 100644 --- a/benchmarks/human_eval_infilling/README.md +++ b/benchmarks/human_eval_infilling/README.md @@ -37,23 +37,23 @@ holds only the dataset definition + prompt + prepare script. ```bash # Prepare benchmark data (downloads all three default splits; # default benchmark variant is random_span) -ng_prepare_benchmark "+config_paths=[benchmarks/human_eval_infilling/config.yaml]" +gym eval prepare --benchmark human_eval_infilling # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/human_eval_infilling/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark human_eval_infilling # Collecting rollouts. The +prompt_config= override is required because # the prepared JSONL stores raw rows (no `responses_create_params.input`); -# ng_collect_rollouts does not pick up the `prompt_config:` field on the -# dataset entry in config.yaml the way ng_run does. -ng_collect_rollouts \ - +agent_name=human_eval_infilling_simple_agent \ - +input_jsonl_fpath=benchmarks/human_eval_infilling/data/random_span.jsonl \ - +prompt_config=benchmarks/human_eval_infilling/prompts/default.yaml \ - +output_jsonl_fpath=results/human_eval_infilling_rollouts.jsonl \ - +num_repeats=1 +# gym eval run --no-serve does not pick up the `prompt_config:` field on the +# dataset entry in config.yaml the way gym env run does. +gym eval run --no-serve \ + --agent human_eval_infilling_simple_agent \ + --input benchmarks/human_eval_infilling/data/random_span.jsonl \ + --prompt-config benchmarks/human_eval_infilling/prompts/default.yaml \ + --output results/human_eval_infilling_rollouts.jsonl \ + --num-repeats 1 ``` ### Other splits @@ -62,13 +62,13 @@ To benchmark a different split, point the resource server at it via the config and feed the matching prepared JSONL: ```bash -ng_collect_rollouts \ - +agent_name=human_eval_infilling_simple_agent \ - '++policy_model.resources_servers.code_fim.split=single_line' \ - +input_jsonl_fpath=benchmarks/human_eval_infilling/data/single_line.jsonl \ - +prompt_config=benchmarks/human_eval_infilling/prompts/default.yaml \ - +output_jsonl_fpath=results/human_eval_infilling_single_line_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent human_eval_infilling_simple_agent \ + --input benchmarks/human_eval_infilling/data/single_line.jsonl \ + --prompt-config benchmarks/human_eval_infilling/prompts/default.yaml \ + --output results/human_eval_infilling_single_line_rollouts.jsonl \ + --num-repeats 1 \ + '++policy_model.resources_servers.code_fim.split=single_line' ``` Start vLLM with `--reasoning-parser ` (e.g. `deepseek_r1` for diff --git a/benchmarks/ifbench/README.md b/benchmarks/ifbench/README.md index 55c5c333c2..2961ecdcd2 100644 --- a/benchmarks/ifbench/README.md +++ b/benchmarks/ifbench/README.md @@ -14,22 +14,23 @@ The `ifbench` resources server clones the AllenAI IFBench repo from GitHub on fi ## Prepare data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/ifbench/config.yaml]" +gym eval prepare --benchmark ifbench ``` ## Run ```bash -ng_e2e_collect_rollouts \ - "+config_paths=[benchmarks/ifbench/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - ++output_jsonl_fpath=results/benchmarks/ifbench.jsonl \ +gym eval run \ + --benchmark ifbench \ + --model-type vllm_model \ + --output results/benchmarks/ifbench.jsonl \ + --split benchmark \ + --model-url <> \ + --model-api-key <> \ + --model <> \ + --resume \ ++overwrite_metrics_conflicts=true \ - ++split=benchmark \ - ++resume_from_cache=true \ - ++reuse_existing_data_preparation=true \ - ++policy_base_url=<> \ - ++policy_api_key=<> \ - ++policy_model_name=<> + ++reuse_existing_data_preparation=true ``` ## No internet access diff --git a/benchmarks/ifeval/README.md b/benchmarks/ifeval/README.md index 891b537178..59035bb84b 100644 --- a/benchmarks/ifeval/README.md +++ b/benchmarks/ifeval/README.md @@ -7,25 +7,25 @@ This benchmark chains to the existing `instruction_following` resources server, ## Prepare data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/ifeval/config.yaml]" +gym eval prepare --benchmark ifeval ``` ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/ifeval/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark ifeval ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=ifeval_instruction_following_simple_agent \ - +input_jsonl_fpath=benchmarks/ifeval/data/ifeval_benchmark.jsonl \ - +output_jsonl_fpath=results/ifeval_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent ifeval_instruction_following_simple_agent \ + --input benchmarks/ifeval/data/ifeval_benchmark.jsonl \ + --output results/ifeval_rollouts.jsonl \ + --num-repeats 4 ``` ## Scoring notes diff --git a/benchmarks/imo_answerbench/README.md b/benchmarks/imo_answerbench/README.md index a2aaf420b4..8fb41f3624 100644 --- a/benchmarks/imo_answerbench/README.md +++ b/benchmarks/imo_answerbench/README.md @@ -7,25 +7,26 @@ Math benchmark from [google-deepmind/superhuman](https://github.com/google-deepm `prepare.py` downloads `answerbench_v2.csv` from the exact pinned superhuman commit that Skills uses, and writes `data/imo_answerbench_benchmark.jsonl` with one row per problem (`question`, `expected_answer`, plus `problem_id` / `category` / `subcategory` / `source`). ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/imo_answerbench/config.yaml]" +gym eval prepare --benchmark imo_answerbench ``` ## Run servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/imo_answerbench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark imo_answerbench ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=imo_answerbench_math_with_autograder_simple_agent \ - +input_jsonl_fpath=benchmarks/imo_answerbench/data/imo_answerbench_benchmark.jsonl \ - +output_jsonl_fpath=results/imo_answerbench_rollouts.jsonl \ - +prompt_config=benchmarks/imo_answerbench/prompts/default.yaml \ - +num_repeats=4 \ +gym eval run --no-serve \ + --agent imo_answerbench_math_with_autograder_simple_agent \ + --input benchmarks/imo_answerbench/data/imo_answerbench_benchmark.jsonl \ + --output results/imo_answerbench_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/imo_answerbench/prompts/default.yaml \ +num_repeats_add_seed=true ``` diff --git a/benchmarks/imo_gradingbench/README.md b/benchmarks/imo_gradingbench/README.md index de46a420b8..dfb4235b2b 100644 --- a/benchmarks/imo_gradingbench/README.md +++ b/benchmarks/imo_gradingbench/README.md @@ -33,20 +33,20 @@ details. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/imo_gradingbench/config.yaml]" +gym eval prepare --benchmark imo_gradingbench # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/imo_gradingbench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark imo_gradingbench # Collecting rollouts (4 rollouts per task) -ng_collect_rollouts \ - +agent_name=imo_gradingbench_imo_gradingbench_simple_agent \ - +input_jsonl_fpath=benchmarks/imo_gradingbench/data/imo_gradingbench_benchmark.jsonl \ - +output_jsonl_fpath=results/imo_gradingbench_rollouts.jsonl \ - +prompt_config=benchmarks/imo_gradingbench/prompts/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent imo_gradingbench_imo_gradingbench_simple_agent \ + --input benchmarks/imo_gradingbench/data/imo_gradingbench_benchmark.jsonl \ + --output results/imo_gradingbench_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/imo_gradingbench/prompts/default.yaml ``` ## Metrics diff --git a/benchmarks/imo_proofbench/README.md b/benchmarks/imo_proofbench/README.md index 3bf4908726..c67c634b6a 100644 --- a/benchmarks/imo_proofbench/README.md +++ b/benchmarks/imo_proofbench/README.md @@ -7,15 +7,15 @@ Math benchmark from [google-deepmind/superhuman](https://github.com/google-deepm `prepare.py` downloads `proofbench.csv` from the exact pinned superhuman commit Skills uses (`c1ee02e03d4cdb2ab21cd01ac927d895f5287fc8`) and writes `data/imo_proofbench_benchmark.jsonl`. Each row carries `problem`, `reference_solution`, `rubric`, plus `problem_id` / `category` / `level` / `expected_answer` / `source`. ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/imo_proofbench/config.yaml]" +gym eval prepare --benchmark imo_proofbench ``` ## Run servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/imo_proofbench/config.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --model-type vllm_model \ + --benchmark imo_proofbench \ +judge_base_url=https://generativelanguage.googleapis.com/v1beta/openai \ "+judge_api_key=$GEMINI_API_KEY" \ +judge_model_name=gemini-2.5-pro @@ -25,16 +25,16 @@ ng_run "+config_paths=[$config_paths]" \ `prepare.py` writes raw problem fields and does **not** bake `responses_create_params.input` into each row. Pass `+prompt_config` so -`ng_collect_rollouts` materialises the chat messages from +`gym eval run --no-serve` materialises the chat messages from `prompts/default.yaml` at rollout time: ```bash -ng_collect_rollouts \ - +agent_name=imo_proofbench_imo_proofbench_judge_simple_agent \ - +prompt_config=benchmarks/imo_proofbench/prompts/default.yaml \ - +input_jsonl_fpath=benchmarks/imo_proofbench/data/imo_proofbench_benchmark.jsonl \ - +output_jsonl_fpath=results/imo_proofbench_rollouts.jsonl \ - +num_repeats=4 \ +gym eval run --no-serve \ + --agent imo_proofbench_imo_proofbench_judge_simple_agent \ + --input benchmarks/imo_proofbench/data/imo_proofbench_benchmark.jsonl \ + --output results/imo_proofbench_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/imo_proofbench/prompts/default.yaml \ +num_repeats_add_seed=true ``` diff --git a/benchmarks/ioi/README.md b/benchmarks/ioi/README.md index 54028ecfef..b4d5ff4540 100644 --- a/benchmarks/ioi/README.md +++ b/benchmarks/ioi/README.md @@ -47,15 +47,21 @@ elsewhere. Cluster/SLURM users can co-launch the sandbox via Skills' ## Running ```bash -ng_prepare_data +config_paths=[benchmarks/ioi/config.yaml] \ - +output_dirpath=benchmarks/ioi/data \ - +mode=benchmark_preparation - -ng_run +config_paths=[benchmarks/ioi/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml] - -ng_collect_rollouts +agent_name=ioi_simple_agent \ - +input_jsonl_fpath=benchmarks/ioi/data/ioi24_benchmark.jsonl \ - +output_jsonl_fpath=results/ioi_rollouts.jsonl \ - +num_repeats=50 +num_repeats_add_seed=true \ - "+responses_create_params={max_output_tokens: 131072, temperature: 1.0, top_p: 0.95}" +gym dataset collate --config benchmarks/ioi/config.yaml \ + --output-dir benchmarks/ioi/data \ + --mode benchmark_preparation + +gym env run \ + --benchmark ioi \ + --model-type vllm_model + +gym eval run --no-serve \ + --agent ioi_simple_agent \ + --input benchmarks/ioi/data/ioi24_benchmark.jsonl \ + --output results/ioi_rollouts.jsonl \ + --num-repeats 50 \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 131072 \ + +num_repeats_add_seed=true ``` diff --git a/benchmarks/labbench2_vlm/README.md b/benchmarks/labbench2_vlm/README.md index c506b91c15..68ceb7fb31 100644 --- a/benchmarks/labbench2_vlm/README.md +++ b/benchmarks/labbench2_vlm/README.md @@ -50,7 +50,7 @@ https://huggingface.co/settings/tokens, and set `hf_token` in `env.yaml` (see above). ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/labbench2_vlm/config.yaml]" +gym eval prepare --benchmark labbench2_vlm ``` Downloads the subtask splits from HuggingFace and media files (images, @@ -82,10 +82,12 @@ prepared in its validation JSONL but is not included in `example.jsonl`. After changing `example.jsonl`, regenerate its static validation metrics: ```bash -.venv/bin/ng_prepare_data \ - "+config_paths=[resources_servers/labbench2_vlm/configs/labbench2_vlm.yaml,resources_servers/labbench2_vlm/configs/judge_model_openai.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - +mode=example_validation \ - +output_dirpath=/tmp/labbench2_vlm_example_validation \ +.venv/bin/gym dataset collate \ + --resource-server labbench2_vlm \ + --config resources_servers/labbench2_vlm/configs/judge_model_openai.yaml \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --mode example_validation \ + --output-dir /tmp/labbench2_vlm_example_validation \ +overwrite_metrics_conflicts=true ``` @@ -98,18 +100,21 @@ overwrite source data. The full config chain is required because ```bash # Start servers -ng_run "+config_paths=[benchmarks/labbench2_vlm/config.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env run \ + --benchmark labbench2_vlm \ + --model-type openai_model # Collect rollouts -ng_collect_rollouts \ - "+config_paths=[benchmarks/labbench2_vlm/config.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - +agent_name=labbench2_vlm_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/labbench2_vlm/data/labbench2_vlm_benchmark.jsonl \ - +output_jsonl_fpath=results/labbench2_vlm.jsonl +gym eval run --no-serve \ + --benchmark labbench2_vlm \ + --model-type openai_model \ + --agent labbench2_vlm_benchmark_simple_agent \ + --input benchmarks/labbench2_vlm/data/labbench2_vlm_benchmark.jsonl \ + --output results/labbench2_vlm.jsonl ``` `+agent_name` and `+input_jsonl_fpath` are both required — rows in the -prepared JSONL don't carry an `agent_ref`, and `ng_collect_rollouts` doesn't +prepared JSONL don't carry an `agent_ref`, and `gym eval run --no-serve` doesn't read the path from the benchmark config. For **protocolqa2** as **extracted text** with a text-capable policy model, pass @@ -124,20 +129,21 @@ PDFs as pages like other PDF tasks. ### One-shot alternative -`ng_e2e_collect_rollouts` starts the server stack, preprocesses, and -collects rollouts in a single command (don't run `ng_run` separately). +`gym eval run` starts the server stack, preprocesses, and +collects rollouts in a single command (don't run `gym env run` separately). Input path and agent ref are auto-derived from the `type: benchmark` dataset entry in the chained config: ```bash -ng_e2e_collect_rollouts \ - "+config_paths=[benchmarks/labbench2_vlm/config.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - ++split=benchmark \ - ++output_jsonl_fpath=results/labbench2_vlm.jsonl \ - +num_samples_in_parallel=16 +gym eval run \ + --benchmark labbench2_vlm \ + --model-type openai_model \ + --split benchmark \ + --output results/labbench2_vlm.jsonl \ + --concurrency 16 ``` -For a fast smoke test, add `+limit=10 +num_repeats=1`. +For a fast smoke test, add `--limit 10 --num-repeats 1`. `num_repeats` defaults to 3. Bump higher for tighter variance on the judge-based reward. @@ -145,14 +151,14 @@ judge-based reward. ## Throttling Each in-flight sample fans out to one policy call + one judge call, so the -endpoints see roughly `2 × num_samples_in_parallel` concurrent requests. +endpoints see roughly `2 × concurrency` concurrent requests. On a hosted endpoint you'll likely hit rate limits or socket errors (`Hit N global ClientOSError`) well before saturating your machine. -Cap concurrency with `+num_samples_in_parallel=`: +Cap concurrency with `--concurrency `: ```bash -ng_collect_rollouts ... +num_samples_in_parallel=16 +gym eval run --no-serve ... --concurrency 16 ``` Start around 16 and bump up if it holds. diff --git a/benchmarks/librispeech_pc/README.md b/benchmarks/librispeech_pc/README.md index a80b5a4019..b70bf6758c 100644 --- a/benchmarks/librispeech_pc/README.md +++ b/benchmarks/librispeech_pc/README.md @@ -8,7 +8,7 @@ which provides the WER scoring. ## Splits This benchmark exposes the `test_clean` split (~2.4k utterances). -`ng_prepare_benchmark` enforces one benchmark dataset per agent, so the +`gym eval prepare` enforces one benchmark dataset per agent, so the harder `test_other` split is left for a sibling benchmark dir as a future PR. `prepare.py` accepts `--splits test-other` on the command line and writes a separate `librispeech_pc_test_other.jsonl` if you want to @@ -34,7 +34,7 @@ row. ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/librispeech_pc/config.yaml]" +gym eval prepare --benchmark librispeech_pc ``` Downloads OpenSLR-145 manifests and OpenSLR-12 test-clean audio @@ -43,18 +43,18 @@ and writes the JSONL into `benchmarks/librispeech_pc/data/`. ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/librispeech_pc/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark librispeech_pc ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=librispeech_pc_asr_with_pc_simple_agent \ - +output_jsonl_fpath=results/librispeech_pc_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent librispeech_pc_asr_with_pc_simple_agent \ + --output results/librispeech_pc_rollouts.jsonl \ + --num-repeats 4 ``` ## Verification diff --git a/benchmarks/livecodebench-x/README.md b/benchmarks/livecodebench-x/README.md index eaefaf4e0e..9bf1f83e34 100644 --- a/benchmarks/livecodebench-x/README.md +++ b/benchmarks/livecodebench-x/README.md @@ -36,7 +36,7 @@ source the existing monolingual `livecodebench/v5_2408_2502` and ## Data Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/livecodebench-x/config.yaml]" +gym eval prepare --benchmark livecodebench-x ``` That writes `benchmarks/livecodebench-x/data/livecodebench-x_benchmark.jsonl` @@ -47,7 +47,7 @@ have the same characteristic; `code_gen.verify()` already handles it. For a smaller subset (e.g. one language × one version, ~300 rows) suitable for local smoke-testing, invoke the prepare script directly with its argparse -flags — `ng_prepare_benchmark` calls `prepare()` with no kwargs and so cannot +flags — `gym eval prepare` calls `prepare()` with no kwargs and so cannot forward these: ```bash @@ -64,26 +64,32 @@ python benchmarks/livecodebench-x/prepare.py --prompt_language en ## Quickstart ```bash -ng_run "+config_paths=[benchmarks/livecodebench-x/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --benchmark livecodebench-x \ + --model-type vllm_model ``` Then in another shell: ```bash mkdir -p results/livecodebench-x -ng_collect_rollouts \ - "+config_paths=[benchmarks/livecodebench-x/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +agent_name=livecodebench-x_code_gen_simple_agent \ - +input_jsonl_fpath=benchmarks/livecodebench-x/data/livecodebench-x_benchmark.jsonl \ - +output_jsonl_fpath=results/livecodebench-x/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml \ - +num_repeats=4 +num_repeats_add_seed=true \ - "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 16384}" +gym eval run --no-serve \ + --benchmark livecodebench-x \ + --model-type vllm_model \ + --agent livecodebench-x_code_gen_simple_agent \ + --input benchmarks/livecodebench-x/data/livecodebench-x_benchmark.jsonl \ + --output results/livecodebench-x/rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/prompts/generic/default.yaml \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 16384 \ + +num_repeats_add_seed=true ``` -`+config_paths` and `+prompt_config` are required: the prepared JSONL ships +`--config` and `+prompt_config` are required: the prepared JSONL ships raw benchmark rows (no `responses_create_params.input` baked in), and the -agent's dataset-level `prompt_config` is metadata for `ng_run` only — the +agent's dataset-level `prompt_config` is metadata for `gym env run` only — the rollout CLI needs `+prompt_config=...` directly to apply the prompt template before merging `responses_create_params` overrides. `mkdir -p` is needed -because `ng_collect_rollouts` does not create parent directories. +because `gym eval run --no-serve` does not create parent directories. diff --git a/benchmarks/longbench_v2/README.md b/benchmarks/longbench_v2/README.md index 61fbd756ba..75aae3081a 100644 --- a/benchmarks/longbench_v2/README.md +++ b/benchmarks/longbench_v2/README.md @@ -42,29 +42,29 @@ python benchmarks/longbench_v2/prepare.py \ ```bash # Prepare benchmark data (default) -ng_prepare_benchmark "+config_paths=[benchmarks/longbench_v2/config.yaml]" +gym eval prepare --benchmark longbench_v2 # Prepare benchmark data (N3 1M variant) -ng_prepare_benchmark "+config_paths=[benchmarks/longbench_v2/config_n3_1m.yaml]" +gym eval prepare --benchmark longbench_v2/config_n3_1m # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/longbench_v2/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark longbench_v2 # Collecting rollouts — default -ng_collect_rollouts \ - +agent_name=longbench_v2_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/longbench_v2/data/longbench_v2_benchmark.jsonl \ - +output_jsonl_fpath=results/longbench_v2_rollouts.jsonl \ - +prompt_config=benchmarks/longbench_v2/prompts/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent longbench_v2_mcqa_simple_agent \ + --input benchmarks/longbench_v2/data/longbench_v2_benchmark.jsonl \ + --output results/longbench_v2_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/longbench_v2/prompts/default.yaml # Collecting rollouts — N3 1M -ng_collect_rollouts \ - +agent_name=longbench_v2_n3_1m_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/longbench_v2/data/longbench_v2_n3_1m_benchmark.jsonl \ - +output_jsonl_fpath=results/longbench_v2_n3_1m_rollouts.jsonl \ - +prompt_config=benchmarks/longbench_v2/prompts/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent longbench_v2_n3_1m_mcqa_simple_agent \ + --input benchmarks/longbench_v2/data/longbench_v2_n3_1m_benchmark.jsonl \ + --output results/longbench_v2_n3_1m_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/longbench_v2/prompts/default.yaml ``` diff --git a/benchmarks/longcodebench/README.md b/benchmarks/longcodebench/README.md index 60192ab366..bf1e17d639 100644 --- a/benchmarks/longcodebench/README.md +++ b/benchmarks/longcodebench/README.md @@ -36,29 +36,29 @@ python benchmarks/longcodebench/prepare.py \ ```bash # Prepare benchmark data (default) -ng_prepare_benchmark "+config_paths=[benchmarks/longcodebench/config.yaml]" +gym eval prepare --benchmark longcodebench # Prepare benchmark data (N3 1M variant) -ng_prepare_benchmark "+config_paths=[benchmarks/longcodebench/config_n3_1m.yaml]" +gym eval prepare --benchmark longcodebench/config_n3_1m # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/longcodebench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark longcodebench # Collecting rollouts — default -ng_collect_rollouts \ - +agent_name=longcodebench_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/longcodebench/data/longcodebench_benchmark.jsonl \ - +output_jsonl_fpath=results/longcodebench_rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent longcodebench_mcqa_simple_agent \ + --input benchmarks/longcodebench/data/longcodebench_benchmark.jsonl \ + --output results/longcodebench_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/prompts/generic/default.yaml # Collecting rollouts — N3 1M -ng_collect_rollouts \ - +agent_name=longcodebench_n3_1m_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/longcodebench/data/longcodebench_n3_1m_benchmark.jsonl \ - +output_jsonl_fpath=results/longcodebench_n3_1m_rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent longcodebench_n3_1m_mcqa_simple_agent \ + --input benchmarks/longcodebench/data/longcodebench_n3_1m_benchmark.jsonl \ + --output results/longcodebench_n3_1m_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/prompts/generic/default.yaml ``` diff --git a/benchmarks/m_arena_hard/README.md b/benchmarks/m_arena_hard/README.md index 062d8e1688..54d1f1f784 100644 --- a/benchmarks/m_arena_hard/README.md +++ b/benchmarks/m_arena_hard/README.md @@ -22,7 +22,7 @@ it back into the prepared JSONL. ## Data Runtime download only — benchmark JSONL is not committed. Run -[`prepare.py`](prepare.py) (or `ng_prepare_benchmark`) to populate +[`prepare.py`](prepare.py) (or `gym eval prepare`) to populate `data/m_arena_hard_benchmark.jsonl`. The prepare script: 1. Calls `datasets.load_dataset("CohereLabs/m-ArenaHard", lang, split="test")` @@ -45,22 +45,22 @@ running `prepare.py`. ```bash # Prepare benchmark data (no baseline -> baseline_answer is empty string) -ng_prepare_benchmark "+config_paths=[benchmarks/m_arena_hard/config.yaml]" +gym eval prepare --benchmark m_arena_hard # Or call prepare.py directly with a baseline JSONL python benchmarks/m_arena_hard/prepare.py --baseline-file path/to/baselines.jsonl # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/m_arena_hard/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark m_arena_hard # Collecting rollouts -ng_collect_rollouts \ - +agent_name=m_arena_hard_arena_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/m_arena_hard/data/m_arena_hard_benchmark.jsonl \ - +output_jsonl_fpath=results/m_arena_hard_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent m_arena_hard_arena_judge_simple_agent \ + --input benchmarks/m_arena_hard/data/m_arena_hard_benchmark.jsonl \ + --output results/m_arena_hard_rollouts.jsonl \ + --num-repeats 4 ``` ## Metrics diff --git a/benchmarks/m_arena_hard_v2/README.md b/benchmarks/m_arena_hard_v2/README.md index bbf7c38208..1e8f2a5201 100644 --- a/benchmarks/m_arena_hard_v2/README.md +++ b/benchmarks/m_arena_hard_v2/README.md @@ -22,7 +22,7 @@ for the judging protocol and metric details. ## Data Runtime download only — benchmark JSONL is not committed. Run -[`prepare.py`](prepare.py) (or `ng_prepare_benchmark`) to populate +[`prepare.py`](prepare.py) (or `gym eval prepare`) to populate `data/m_arena_hard_v2_benchmark.jsonl`. The prepare script loads the HF dataset across all 23 language configs, iterates each split, and emits one row per `(language, question_id)` with `uid`, `question`, @@ -55,19 +55,19 @@ python benchmarks/m_arena_hard_v2/prepare.py \ ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/m_arena_hard_v2/config.yaml]" +gym eval prepare --benchmark m_arena_hard_v2 # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/m_arena_hard_v2/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark m_arena_hard_v2 # Collecting rollouts -ng_collect_rollouts \ - +agent_name=m_arena_hard_v2_arena_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/m_arena_hard_v2/data/m_arena_hard_v2_benchmark.jsonl \ - +output_jsonl_fpath=results/m_arena_hard_v2_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent m_arena_hard_v2_arena_judge_simple_agent \ + --input benchmarks/m_arena_hard_v2/data/m_arena_hard_v2_benchmark.jsonl \ + --output results/m_arena_hard_v2_rollouts.jsonl \ + --num-repeats 4 ``` ## Metrics diff --git a/benchmarks/math-500/README.md b/benchmarks/math-500/README.md index fdebaa0226..40df2cc50f 100644 --- a/benchmarks/math-500/README.md +++ b/benchmarks/math-500/README.md @@ -15,17 +15,17 @@ resource server. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/math-500/config.yaml]" +gym eval prepare --benchmark math-500 # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/math-500/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark math-500 # Collecting rollouts -ng_collect_rollouts \ - +agent_name=math_500_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/math-500/data/math-500_benchmark.jsonl \ - +output_jsonl_fpath=results/math-500/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/math.yaml +gym eval run --no-serve \ + --agent math_500_math_with_judge_simple_agent \ + --input benchmarks/math-500/data/math-500_benchmark.jsonl \ + --output results/math-500/rollouts.jsonl \ + --prompt-config benchmarks/prompts/generic/math.yaml ``` diff --git a/benchmarks/mbpp/README.md b/benchmarks/mbpp/README.md index 45859a3c69..5461104f2b 100644 --- a/benchmarks/mbpp/README.md +++ b/benchmarks/mbpp/README.md @@ -22,23 +22,23 @@ Verification runs in the `evalplus` resource server (shared with ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/mbpp/config.yaml]" +gym eval prepare --benchmark mbpp # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/mbpp/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark mbpp # Collecting rollouts. The +prompt_config= override is required because # the prepared JSONL stores raw `question` rows (no `responses_create_params.input`); -# ng_collect_rollouts does not pick up the `prompt_config:` field on the dataset -# entry in config.yaml the way ng_run does. -ng_collect_rollouts \ - +agent_name=mbpp_evalplus_simple_agent \ - +input_jsonl_fpath=benchmarks/mbpp/data/mbpp_benchmark.jsonl \ - +prompt_config=benchmarks/mbpp/prompts/default.yaml \ - +output_jsonl_fpath=results/mbpp_rollouts.jsonl \ - +num_repeats=4 +# gym eval run --no-serve does not pick up the `prompt_config:` field on the dataset +# entry in config.yaml the way gym env run does. +gym eval run --no-serve \ + --agent mbpp_evalplus_simple_agent \ + --input benchmarks/mbpp/data/mbpp_benchmark.jsonl \ + --output results/mbpp_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/mbpp/prompts/default.yaml ``` Start vLLM with `--reasoning-parser ` (e.g. `deepseek_r1` for diff --git a/benchmarks/minif2f/README.md b/benchmarks/minif2f/README.md index 6569256f2b..9941d01a6e 100644 --- a/benchmarks/minif2f/README.md +++ b/benchmarks/minif2f/README.md @@ -13,7 +13,7 @@ evaluation protocol). ## Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/minif2f/config.yaml]" +gym eval prepare --benchmark minif2f ``` Downloads the source JSONL, splits header from theorem body, strips `sorry` @@ -30,21 +30,22 @@ and set `NEMO_SKILLS_SANDBOX_HOST` / `NEMO_SKILLS_SANDBOX_PORT` before starting the server. ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/minif2f/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark minif2f ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=minif2f_math_formal_lean_simple_agent \ - +input_jsonl_fpath=benchmarks/minif2f/data/minif2f_benchmark.jsonl \ - +output_jsonl_fpath=results/minif2f_rollouts.jsonl \ - +num_repeats=32 \ - +prompt_config=benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent minif2f_math_formal_lean_simple_agent \ + --input benchmarks/minif2f/data/minif2f_benchmark.jsonl \ + --output results/minif2f_rollouts.jsonl \ + --num-repeats 32 \ + --prompt-config benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml \ + --temperature 1.0 \ + --max-output-tokens 16384 ``` Reproduce published miniF2F numbers on a DeepSeek-Prover / Goedel-Prover class diff --git a/benchmarks/mmlu-redux/README.md b/benchmarks/mmlu-redux/README.md index 5c47f85fbc..8a63979880 100644 --- a/benchmarks/mmlu-redux/README.md +++ b/benchmarks/mmlu-redux/README.md @@ -15,17 +15,17 @@ Migrates NeMo Skills' `mmlu-redux` benchmark to Gym on top of the shared ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/mmlu-redux/config.yaml]" +gym eval prepare --benchmark mmlu-redux # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/mmlu-redux/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark mmlu-redux # Collecting rollouts -ng_collect_rollouts \ - +agent_name=mmlu-redux_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/mmlu-redux/data/mmlu-redux_benchmark.jsonl \ - +output_jsonl_fpath=results/mmlu-redux/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/general-boxed.yaml +gym eval run --no-serve \ + --agent mmlu-redux_mcqa_simple_agent \ + --input benchmarks/mmlu-redux/data/mmlu-redux_benchmark.jsonl \ + --output results/mmlu-redux/rollouts.jsonl \ + --prompt-config benchmarks/prompts/generic/general-boxed.yaml ``` diff --git a/benchmarks/mmlu/README.md b/benchmarks/mmlu/README.md index 6987bf93a3..33fd5839a8 100644 --- a/benchmarks/mmlu/README.md +++ b/benchmarks/mmlu/README.md @@ -14,17 +14,17 @@ resource server. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/mmlu/config.yaml]" +gym eval prepare --benchmark mmlu # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/mmlu/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark mmlu # Collecting rollouts -ng_collect_rollouts \ - +agent_name=mmlu_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/mmlu/data/mmlu_benchmark.jsonl \ - +output_jsonl_fpath=results/mmlu/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/eval/aai/mcq-4choices-boxed.yaml +gym eval run --no-serve \ + --agent mmlu_mcqa_simple_agent \ + --input benchmarks/mmlu/data/mmlu_benchmark.jsonl \ + --output results/mmlu/rollouts.jsonl \ + --prompt-config benchmarks/prompts/eval/aai/mcq-4choices-boxed.yaml ``` diff --git a/benchmarks/mmlu_pro/README.md b/benchmarks/mmlu_pro/README.md index a61aa8880c..2a0be5e720 100644 --- a/benchmarks/mmlu_pro/README.md +++ b/benchmarks/mmlu_pro/README.md @@ -13,13 +13,16 @@ This benchmark uses the `mcqa` resource server with the `mcqa_simple_agent`. ```bash # Prepare data -ng_prepare_benchmark "+config_paths=[benchmarks/mmlu_pro/config.yaml]" +gym eval prepare --benchmark mmlu_pro # Start servers -ng_run "+config_paths=[benchmarks/mmlu_pro/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --benchmark mmlu_pro \ + --model-type vllm_model # Collect rollouts -ng_collect_rollouts \ - "+config_paths=[benchmarks/mmlu_pro/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +output_jsonl_fpath=results/mmlu_pro.jsonl +gym eval run --no-serve \ + --benchmark mmlu_pro \ + --model-type vllm_model \ + --output results/mmlu_pro.jsonl ``` diff --git a/benchmarks/mmlu_prox/README.md b/benchmarks/mmlu_prox/README.md index 35676c43a0..72c5cab8f8 100644 --- a/benchmarks/mmlu_prox/README.md +++ b/benchmarks/mmlu_prox/README.md @@ -13,13 +13,16 @@ This benchmark uses the `mcqa` resource server with the `mcqa_simple_agent`. ```bash # Prepare data -ng_prepare_benchmark "+config_paths=[benchmarks/mmlu_prox/config.yaml]" +gym eval prepare --benchmark mmlu_prox # Start servers -ng_run "+config_paths=[benchmarks/mmlu_prox/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --benchmark mmlu_prox \ + --model-type vllm_model # Collect rollouts -ng_collect_rollouts \ - "+config_paths=[benchmarks/mmlu_prox/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +output_jsonl_fpath=results/mmlu_prox.jsonl +gym eval run --no-serve \ + --benchmark mmlu_prox \ + --model-type vllm_model \ + --output results/mmlu_prox.jsonl ``` diff --git a/benchmarks/mmmlu/README.md b/benchmarks/mmmlu/README.md index bae74b2969..93f80f5768 100644 --- a/benchmarks/mmmlu/README.md +++ b/benchmarks/mmmlu/README.md @@ -14,17 +14,17 @@ resource server. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/mmmlu/config.yaml]" +gym eval prepare --benchmark mmmlu # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/mmmlu/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark mmmlu # Collecting rollouts -ng_collect_rollouts \ - +agent_name=mmmlu_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/mmmlu/data/mmmlu_benchmark.jsonl \ - +output_jsonl_fpath=results/mmmlu/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml +gym eval run --no-serve \ + --agent mmmlu_mcqa_simple_agent \ + --input benchmarks/mmmlu/data/mmmlu_benchmark.jsonl \ + --output results/mmmlu/rollouts.jsonl \ + --prompt-config benchmarks/prompts/generic/default.yaml ``` diff --git a/benchmarks/mobench/README.md b/benchmarks/mobench/README.md index b32706e6a1..e2e9c91d19 100644 --- a/benchmarks/mobench/README.md +++ b/benchmarks/mobench/README.md @@ -12,7 +12,7 @@ and `simple_agent` (single-turn, matching NeMo-Skills' evaluation protocol). ## Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/mobench/config.yaml]" +gym eval prepare --benchmark mobench ``` Downloads the source JSONL, splits the prelude from the theorem block via regex, @@ -29,21 +29,22 @@ and set `NEMO_SKILLS_SANDBOX_HOST` / `NEMO_SKILLS_SANDBOX_PORT` before starting the server. ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/mobench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark mobench ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=mobench_math_formal_lean_simple_agent \ - +input_jsonl_fpath=benchmarks/mobench/data/mobench_benchmark.jsonl \ - +output_jsonl_fpath=results/mobench_rollouts.jsonl \ - +num_repeats=32 \ - +prompt_config=benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent mobench_math_formal_lean_simple_agent \ + --input benchmarks/mobench/data/mobench_benchmark.jsonl \ + --output results/mobench_rollouts.jsonl \ + --num-repeats 32 \ + --prompt-config benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml \ + --temperature 1.0 \ + --max-output-tokens 16384 ``` Reproduce published MOBench numbers on a DeepSeek-Prover / Goedel-Prover class diff --git a/benchmarks/mrcr/README.md b/benchmarks/mrcr/README.md index d31ae8ec0c..8e89886e2e 100644 --- a/benchmarks/mrcr/README.md +++ b/benchmarks/mrcr/README.md @@ -33,44 +33,46 @@ python benchmarks/mrcr/prepare.py \ ```bash # Default (o200k_base, no filter) -ng_prepare_benchmark "+config_paths=[benchmarks/mrcr/config.yaml]" +gym eval prepare --benchmark mrcr # N3 128k variant -ng_prepare_benchmark "+config_paths=[benchmarks/mrcr/config_n3_128k.yaml]" +gym eval prepare --benchmark mrcr/config_n3_128k # N3 1M variant -ng_prepare_benchmark "+config_paths=[benchmarks/mrcr/config_n3_1m.yaml]" +gym eval prepare --benchmark mrcr/config_n3_1m ``` ## Start environment ```bash -ng_run "+config_paths=[benchmarks/mrcr/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --benchmark mrcr \ + --model-type vllm_model ``` ## Collect rollouts ```bash # Default variant -ng_collect_rollouts \ - +agent_name=mrcr_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/mrcr/data/mrcr_benchmark.jsonl \ - +output_jsonl_fpath=results/mrcr_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent mrcr_benchmark_simple_agent \ + --input benchmarks/mrcr/data/mrcr_benchmark.jsonl \ + --output results/mrcr_rollouts.jsonl \ + --num-repeats 4 # N3 128k variant -ng_collect_rollouts \ - +agent_name=mrcr_n3_128k_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/mrcr/data/mrcr_n3_128k_benchmark.jsonl \ - +output_jsonl_fpath=results/mrcr_n3_128k_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent mrcr_n3_128k_benchmark_simple_agent \ + --input benchmarks/mrcr/data/mrcr_n3_128k_benchmark.jsonl \ + --output results/mrcr_n3_128k_rollouts.jsonl \ + --num-repeats 4 # N3 1M variant -ng_collect_rollouts \ - +agent_name=mrcr_n3_1m_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/mrcr/data/mrcr_n3_1m_benchmark.jsonl \ - +output_jsonl_fpath=results/mrcr_n3_1m_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent mrcr_n3_1m_benchmark_simple_agent \ + --input benchmarks/mrcr/data/mrcr_n3_1m_benchmark.jsonl \ + --output results/mrcr_n3_1m_rollouts.jsonl \ + --num-repeats 4 ``` ## Metrics diff --git a/benchmarks/musan/README.md b/benchmarks/musan/README.md index 4ef8ee8a9d..8a9f86f565 100644 --- a/benchmarks/musan/README.md +++ b/benchmarks/musan/README.md @@ -40,7 +40,7 @@ row. ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/musan/config.yaml]" +gym eval prepare --benchmark musan ``` Downloads the MUSAN OpenSLR archive, extracts it under @@ -53,18 +53,18 @@ points at `/data/musan//audio/musan__NNNNNN.wav` by default ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/musan/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark musan ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=musan_asr_with_pc_simple_agent \ - +output_jsonl_fpath=results/musan_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent musan_asr_with_pc_simple_agent \ + --output results/musan_rollouts.jsonl \ + --num-repeats 4 ``` ## Verification diff --git a/benchmarks/nemotron_3_ultra/README.md b/benchmarks/nemotron_3_ultra/README.md index 7f95ce688c..3aee2957fe 100644 --- a/benchmarks/nemotron_3_ultra/README.md +++ b/benchmarks/nemotron_3_ultra/README.md @@ -36,8 +36,7 @@ Qwen3-235B-A22B-Instruct-2507-FP8: # Prepare benchmark data ```bash -config_paths="benchmarks/nemotron_3_ultra/config_short.yaml" -ng_prepare_benchmark "+config_paths=[$config_paths]" +gym eval prepare --config benchmarks/nemotron_3_ultra/config_short.yaml ``` # Run @@ -65,10 +64,9 @@ This example uses: WANDB_PROJECT=<> EXPERIMENT_NAME=<> -config_paths="benchmarks/nemotron_3_ultra/remote_endpoint.yaml,\ -benchmarks/nemotron_3_ultra/config_short.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ +gym eval run \ + --config benchmarks/nemotron_3_ultra/remote_endpoint.yaml \ + --config benchmarks/nemotron_3_ultra/config_short.yaml \ +wandb_project=$WANDB_PROJECT \ +wandb_name=$EXPERIMENT_NAME \ ++output_jsonl_fpath=results/$EXPERIMENT_NAME.jsonl \ @@ -86,10 +84,9 @@ ng_e2e_collect_rollouts \ WANDB_PROJECT=<> EXPERIMENT_NAME=<> -config_paths="benchmarks/nemotron_3_ultra/local_endpoint.yaml,\ -benchmarks/nemotron_3_ultra/config_short.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ +gym eval run \ + --config benchmarks/nemotron_3_ultra/local_endpoint.yaml \ + --config benchmarks/nemotron_3_ultra/config_short.yaml \ +wandb_project=$WANDB_PROJECT \ +wandb_name=$EXPERIMENT_NAME \ ++output_jsonl_fpath=results/$EXPERIMENT_NAME.jsonl \ @@ -105,10 +102,9 @@ ng_e2e_collect_rollouts \ WANDB_PROJECT=<> EXPERIMENT_NAME=<> -config_paths="benchmarks/nemotron_3_ultra/local_endpoint_no_gpus.yaml,\ -benchmarks/nemotron_3_ultra/config_short_no_gpus.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ +gym eval run \ + --config benchmarks/nemotron_3_ultra/local_endpoint_no_gpus.yaml \ + --config benchmarks/nemotron_3_ultra/config_short_no_gpus.yaml \ +wandb_project=$WANDB_PROJECT \ +wandb_name=$EXPERIMENT_NAME \ ++output_jsonl_fpath=results/$EXPERIMENT_NAME.jsonl \ diff --git a/benchmarks/numb3rs/README.md b/benchmarks/numb3rs/README.md index 206fea0aff..75e3b96fc3 100644 --- a/benchmarks/numb3rs/README.md +++ b/benchmarks/numb3rs/README.md @@ -54,7 +54,7 @@ row. ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/numb3rs/config.yaml]" +gym eval prepare --benchmark numb3rs ``` Downloads `nvidia/Numb3rs` (split=`test`), iterates the 12 categories, @@ -64,18 +64,18 @@ combined `benchmarks/numb3rs/data/numb3rs_benchmark.jsonl`. ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/numb3rs/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark numb3rs ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=numb3rs_asr_with_pc_simple_agent \ - +output_jsonl_fpath=results/numb3rs_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent numb3rs_asr_with_pc_simple_agent \ + --output results/numb3rs_rollouts.jsonl \ + --num-repeats 4 ``` ## Verification diff --git a/benchmarks/omniscience/README.md b/benchmarks/omniscience/README.md index 725375140e..122b45f2c7 100644 --- a/benchmarks/omniscience/README.md +++ b/benchmarks/omniscience/README.md @@ -10,12 +10,13 @@ Tests factual recall across 6 domains: Humanities, Health, Software Engineering, ```bash # Prepare data -ng_prepare_data +benchmark=omniscience +gym dataset collate +benchmark=omniscience # Run benchmark -ng_collect_rollouts +benchmark=omniscience \ - "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +output_jsonl_fpath=results/omniscience_rollouts.jsonl +gym eval run --no-serve \ + --model-type vllm_model \ + --output results/omniscience_rollouts.jsonl \ + +benchmark=omniscience ``` ## Key Metrics diff --git a/benchmarks/physics/README.md b/benchmarks/physics/README.md index 99057c0e23..413c09f7c0 100644 --- a/benchmarks/physics/README.md +++ b/benchmarks/physics/README.md @@ -12,7 +12,7 @@ expression, list, set, or option label. ## Prepare data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/physics/config.yaml]" +gym eval prepare --benchmark physics ``` `prepare.py` downloads the dataset, applies the same flatten-and-`\boxed{}` @@ -24,9 +24,9 @@ transformation Skills uses for the multi-part answers, and writes ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/physics/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark physics ``` > Reasoning-model note: start the policy vLLM server with @@ -37,11 +37,11 @@ ng_run "+config_paths=[$config_paths]" ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=physics_physics_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/physics/data/physics_benchmark.jsonl \ - +output_jsonl_fpath=results/physics_rollouts.jsonl \ - +num_repeats=4 \ +gym eval run --no-serve \ + --agent physics_physics_judge_simple_agent \ + --input benchmarks/physics/data/physics_benchmark.jsonl \ + --output results/physics_rollouts.jsonl \ + --num-repeats 4 \ +num_repeats_add_seed=true ``` diff --git a/benchmarks/polymath/README.md b/benchmarks/polymath/README.md index d1606eab3a..fd711cb29a 100644 --- a/benchmarks/polymath/README.md +++ b/benchmarks/polymath/README.md @@ -38,7 +38,7 @@ matching Skills. ## Data preparation ``` -ng_prepare_benchmark "+config_paths=[benchmarks/polymath/config.yaml]" +gym eval prepare --benchmark polymath ``` Writes `data/polymath_benchmark.jsonl` with one row per @@ -50,20 +50,20 @@ Start the servers (inherits the `polymath` resources server in symbolic-only mode plus a vLLM model server): ``` -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/polymath/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark polymath ``` Collect rollouts (use `+num_repeats=4` for a quick parity pass, `+num_repeats=16` for parity-grade evaluation): ``` -ng_collect_rollouts \ - +agent_name=polymath_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/polymath/data/polymath_benchmark.jsonl \ - +output_jsonl_fpath=results/polymath_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent polymath_benchmark_simple_agent \ + --input benchmarks/polymath/data/polymath_benchmark.jsonl \ + --output results/polymath_rollouts.jsonl \ + --num-repeats 4 ``` Start the model server with `--reasoning-parser ` (e.g. diff --git a/benchmarks/proof-arena-judge/README.md b/benchmarks/proof-arena-judge/README.md index 19ea6e9456..b87f4e7cea 100644 --- a/benchmarks/proof-arena-judge/README.md +++ b/benchmarks/proof-arena-judge/README.md @@ -17,17 +17,17 @@ Adds the `proof-arena-judge` benchmark to Gym. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/proof-arena-judge/config.yaml]" +gym eval prepare --benchmark proof-arena-judge # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/proof-arena-judge/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark proof-arena-judge # Collecting rollouts -ng_collect_rollouts \ - +agent_name=proof_arena_judge_math_proof_judgement_simple_agent \ - +input_jsonl_fpath=benchmarks/proof-arena-judge/data/proof-arena-judge_benchmark.jsonl \ - +output_jsonl_fpath=results/proof-arena-judge/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/judge/math-proof-judge.yaml +gym eval run --no-serve \ + --agent proof_arena_judge_math_proof_judgement_simple_agent \ + --input benchmarks/proof-arena-judge/data/proof-arena-judge_benchmark.jsonl \ + --output results/proof-arena-judge/rollouts.jsonl \ + --prompt-config benchmarks/prompts/judge/math-proof-judge.yaml ``` diff --git a/benchmarks/proof_bench_judge/README.md b/benchmarks/proof_bench_judge/README.md index 0320517a22..38b4e6f8aa 100644 --- a/benchmarks/proof_bench_judge/README.md +++ b/benchmarks/proof_bench_judge/README.md @@ -34,19 +34,19 @@ details. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/proof_bench_judge/config.yaml]" +gym eval prepare --benchmark proof_bench_judge # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/proof_bench_judge/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark proof_bench_judge # Collecting rollouts (4 rollouts per task) -ng_collect_rollouts \ - +agent_name=proof_bench_judge_math_proof_judgement_simple_agent \ - +input_jsonl_fpath=benchmarks/proof_bench_judge/data/proof_bench_judge_benchmark.jsonl \ - +output_jsonl_fpath=results/proof_bench_judge_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent proof_bench_judge_math_proof_judgement_simple_agent \ + --input benchmarks/proof_bench_judge/data/proof_bench_judge_benchmark.jsonl \ + --output results/proof_bench_judge_rollouts.jsonl \ + --num-repeats 4 ``` ## Metrics diff --git a/benchmarks/proofnet/README.md b/benchmarks/proofnet/README.md index cc80c352eb..46ff2bd60a 100644 --- a/benchmarks/proofnet/README.md +++ b/benchmarks/proofnet/README.md @@ -12,7 +12,7 @@ and `simple_agent` (single-turn, matching NeMo-Skills' evaluation protocol). ## Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/proofnet/config.yaml]" +gym eval prepare --benchmark proofnet ``` Downloads the source JSONL, filters to `split == "test"`, and writes @@ -28,19 +28,20 @@ and set `NEMO_SKILLS_SANDBOX_HOST` / `NEMO_SKILLS_SANDBOX_PORT` before starting the server. ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/proofnet/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark proofnet ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=proofnet_math_formal_lean_simple_agent \ - +input_jsonl_fpath=benchmarks/proofnet/data/proofnet_benchmark.jsonl \ - +output_jsonl_fpath=results/proofnet_rollouts.jsonl \ - +num_repeats=32 \ - +prompt_config=benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent proofnet_math_formal_lean_simple_agent \ + --input benchmarks/proofnet/data/proofnet_benchmark.jsonl \ + --output results/proofnet_rollouts.jsonl \ + --num-repeats 32 \ + --max-output-tokens 16384 \ + --temperature 1.0 \ + --prompt-config benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml ``` diff --git a/benchmarks/putnam_bench/README.md b/benchmarks/putnam_bench/README.md index 924b067a5d..dc20c6c8ce 100644 --- a/benchmarks/putnam_bench/README.md +++ b/benchmarks/putnam_bench/README.md @@ -12,7 +12,7 @@ and `simple_agent` (single-turn, matching NeMo-Skills' evaluation protocol). ## Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/putnam_bench/config.yaml]" +gym eval prepare --benchmark putnam_bench ``` Clones the upstream repo into a temp dir, runs its `rewrite_solutions.py` subprocess, parses the output `.lean` files, and writes `data/putnam_bench_benchmark.jsonl`. Each row has `name`, `split`, `header`, `informal_prefix`, `formal_statement` (no `goal` field — the `math_formal_lean` server doesn't use it, and the upstream prepare omits it). @@ -28,19 +28,20 @@ and set `NEMO_SKILLS_SANDBOX_HOST` / `NEMO_SKILLS_SANDBOX_PORT` before starting the server. ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/putnam_bench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark putnam_bench ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=putnam_bench_math_formal_lean_simple_agent \ - +input_jsonl_fpath=benchmarks/putnam_bench/data/putnam_bench_benchmark.jsonl \ - +output_jsonl_fpath=results/putnam_bench_rollouts.jsonl \ - +num_repeats=32 \ - +prompt_config=benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent putnam_bench_math_formal_lean_simple_agent \ + --input benchmarks/putnam_bench/data/putnam_bench_benchmark.jsonl \ + --output results/putnam_bench_rollouts.jsonl \ + --num-repeats 32 \ + --max-output-tokens 16384 \ + --temperature 1.0 \ + --prompt-config benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml ``` diff --git a/benchmarks/ruler/README.md b/benchmarks/ruler/README.md index 61a9257340..4cac5fc118 100644 --- a/benchmarks/ruler/README.md +++ b/benchmarks/ruler/README.md @@ -5,22 +5,20 @@ Linux: `apt update && apt install -y git-lfs` # Prepare data ```bash -config_paths="benchmarks/ruler/config_nemotron_3_256k.yaml" -ng_prepare_benchmark "+config_paths=[$config_paths]" +gym eval prepare --benchmark ruler/config_nemotron_3_256k ``` # Run ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/ruler/config_nemotron_3_256k.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++output_jsonl_fpath=results/benchmarks/ruler.jsonl \ +gym eval run \ + --model-type vllm_model \ + --benchmark ruler/config_nemotron_3_256k \ + --output results/benchmarks/ruler.jsonl \ + --split benchmark \ + --model-url <> \ + --model-api-key <> \ + --model <> \ + --resume \ ++overwrite_metrics_conflicts=true \ - ++split=benchmark \ - ++resume_from_cache=true \ - ++reuse_existing_data_preparation=true \ - ++policy_base_url=<> \ - ++policy_api_key=<> \ - ++policy_model_name=<> + ++reuse_existing_data_preparation=true ``` diff --git a/benchmarks/simpleqa/README.md b/benchmarks/simpleqa/README.md index 8bd24dde26..7bb4512fdb 100644 --- a/benchmarks/simpleqa/README.md +++ b/benchmarks/simpleqa/README.md @@ -34,26 +34,26 @@ and derives `f1` + `accuracy_given_attempted` at every aggregation level. ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/simpleqa/config.yaml]" +gym eval prepare --benchmark simpleqa ``` ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/simpleqa/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark simpleqa ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=simpleqa_simpleqa_simple_agent \ - +input_jsonl_fpath=benchmarks/simpleqa/data/simpleqa_benchmark.jsonl \ - +output_jsonl_fpath=results/simpleqa_rollouts.jsonl \ - +prompt_config=benchmarks/simpleqa/prompts/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent simpleqa_simpleqa_simple_agent \ + --input benchmarks/simpleqa/data/simpleqa_benchmark.jsonl \ + --output results/simpleqa_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/simpleqa/prompts/default.yaml ``` For reasoning models, start the policy vLLM server with diff --git a/benchmarks/speed-bench/README.md b/benchmarks/speed-bench/README.md index fc7f68db90..e247232263 100644 --- a/benchmarks/speed-bench/README.md +++ b/benchmarks/speed-bench/README.md @@ -51,23 +51,23 @@ configurations: # Prepare benchmark data (downloads the upstream HF dataset # nvidia/SPEED-Bench plus the 14 source datasets it interpolates from). # Run on a host that has internet access — see prepare.py for details. -ng_prepare_benchmark "+config_paths=[benchmarks/speed-bench/config_qualitative.yaml]" +gym eval prepare --benchmark speed-bench/config_qualitative # Running servers — uses the local_vllm_model demo config that bakes # ngram speculative decoding into vllm_serve_kwargs.speculative_config. # To use a different target model, swap this for any local_vllm_model # config that includes a `speculative_config:` block. -config_paths="responses_api_models/local_vllm_model/configs/Qwen/Qwen3-30B-A3B-Instruct-2507-ngram-specdec.yaml,\ -benchmarks/speed-bench/config_qualitative.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --model-type local_vllm_model/Qwen/Qwen3-30B-A3B-Instruct-2507-ngram-specdec \ + --benchmark speed-bench/config_qualitative \ +policy_model=Qwen3-30B-A3B-Instruct-2507-ngram-specdec # Collecting rollouts -ng_collect_rollouts \ - +agent_name=speed_bench_qualitative_simple_agent \ - +input_jsonl_fpath=benchmarks/speed-bench/data/speed_bench_qualitative_benchmark.jsonl \ - +output_jsonl_fpath=results/speed_bench_qualitative_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent speed_bench_qualitative_simple_agent \ + --input benchmarks/speed-bench/data/speed_bench_qualitative_benchmark.jsonl \ + --output results/speed_bench_qualitative_rollouts.jsonl \ + --num-repeats 1 ``` If you're using the lighter-weight `vllm_model` config (external vLLM diff --git a/benchmarks/spider2_lite/README.md b/benchmarks/spider2_lite/README.md index cdcd7e03f3..5846b3ccbd 100644 --- a/benchmarks/spider2_lite/README.md +++ b/benchmarks/spider2_lite/README.md @@ -1,22 +1,20 @@ # Prepare data ```bash -config_paths="benchmarks/spider2_lite/config.yaml" -ng_prepare_benchmark "+config_paths=[$config_paths]" +gym eval prepare --benchmark spider2_lite ``` # Run ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/spider2_lite/config.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++output_jsonl_fpath=results/benchmarks/spider2_lite.jsonl \ +gym eval run \ + --model-type vllm_model \ + --benchmark spider2_lite \ + --output results/benchmarks/spider2_lite.jsonl \ + --split benchmark \ + --model-url <> \ + --model-api-key <> \ + --model <> \ + --resume \ ++overwrite_metrics_conflicts=true \ - ++split=benchmark \ ++spider2_lite_benchmark_resources_server.resources_servers.spider2_lite.max_concurrency=8 \ - ++resume_from_cache=true \ - ++reuse_existing_data_preparation=true \ - ++policy_base_url=<> \ - ++policy_api_key=<> \ - ++policy_model_name=<> + ++reuse_existing_data_preparation=true ``` diff --git a/benchmarks/supergpqa/README.md b/benchmarks/supergpqa/README.md index ff3392f728..66c4462499 100644 --- a/benchmarks/supergpqa/README.md +++ b/benchmarks/supergpqa/README.md @@ -14,17 +14,17 @@ server. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/supergpqa/config.yaml]" +gym eval prepare --benchmark supergpqa # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/supergpqa/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark supergpqa # Collecting rollouts -ng_collect_rollouts \ - +agent_name=supergpqa_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/supergpqa/data/supergpqa_benchmark.jsonl \ - +output_jsonl_fpath=results/supergpqa/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/eval/aai/mcq-10choices.yaml +gym eval run --no-serve \ + --agent supergpqa_mcqa_simple_agent \ + --input benchmarks/supergpqa/data/supergpqa_benchmark.jsonl \ + --output results/supergpqa/rollouts.jsonl \ + --prompt-config benchmarks/prompts/eval/aai/mcq-10choices.yaml ``` diff --git a/benchmarks/ugphysics/README.md b/benchmarks/ugphysics/README.md index a620490c21..1b1248307c 100644 --- a/benchmarks/ugphysics/README.md +++ b/benchmarks/ugphysics/README.md @@ -61,19 +61,21 @@ message. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/ugphysics/config.yaml]" +gym eval prepare --benchmark ugphysics # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/ugphysics/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --benchmark ugphysics # Collecting rollouts -ng_collect_rollouts \ - +agent_name=ugphysics_ugphysics_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/ugphysics/data/ugphysics_benchmark.jsonl \ - +output_jsonl_fpath=results/ugphysics_rollouts.jsonl \ - +prompt_config=benchmarks/ugphysics/prompts/default.yaml \ - +num_repeats=4 \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0, top_p: 0.95}" +gym eval run --no-serve \ + --agent ugphysics_ugphysics_judge_simple_agent \ + --input benchmarks/ugphysics/data/ugphysics_benchmark.jsonl \ + --output results/ugphysics_rollouts.jsonl \ + --num-repeats 4 \ + --max-output-tokens 16384 \ + --temperature 1.0 \ + --top-p 0.95 \ + --prompt-config benchmarks/ugphysics/prompts/default.yaml ``` diff --git a/benchmarks/wmt24pp/README.md b/benchmarks/wmt24pp/README.md index 92022c3ecf..2b2839e11b 100644 --- a/benchmarks/wmt24pp/README.md +++ b/benchmarks/wmt24pp/README.md @@ -14,7 +14,7 @@ details and the Ray GPU-scheduled COMET path. ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/wmt24pp/config.yaml]" +gym eval prepare --benchmark wmt24pp ``` In addition to writing `data/wmt24pp_benchmark.jsonl`, the prepare step @@ -32,21 +32,21 @@ runs disable COMET via Hydra override and rely on corpus-BLEU only; xCOMET scoring still works end-to-end on the cluster path: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/wmt24pp/config.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --model-type vllm_model \ + --benchmark wmt24pp \ "++wmt24pp_wmt_translation_resources_server.resources_servers.wmt_translation.compute_comet=false" ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=wmt24pp_wmt_translation_simple_agent \ - +prompt_config=benchmarks/wmt24pp/prompts/default.yaml \ - +input_jsonl_fpath=benchmarks/wmt24pp/data/wmt24pp_benchmark.jsonl \ - +output_jsonl_fpath=results/wmt24pp_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent wmt24pp_wmt_translation_simple_agent \ + --input benchmarks/wmt24pp/data/wmt24pp_benchmark.jsonl \ + --output results/wmt24pp_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/wmt24pp/prompts/default.yaml ``` ## End-to-end reproduction on a SLURM cluster (via NeMo-Skills) @@ -98,7 +98,7 @@ The two container fields that aren't trivial: #### Prepare benchmark data on the cluster -The local `ng_prepare_benchmark` from [above](#prepare-benchmark-data) +The local `gym eval prepare` from [above](#prepare-benchmark-data) writes the JSONL to your dev workstation. For a SLURM run, the JSONL plus the `Unbabel/XCOMET-XXL` cache need to live on the cluster's filesystem. Dispatch the prepare via `ns run_cmd` with the `nemo-gym` @@ -110,7 +110,7 @@ ns run_cmd \ --cluster \ --container nemo-gym \ --expname wmt24pp_prepare \ - --command 'ng_prepare_benchmark "+config_paths=[benchmarks/wmt24pp/config.yaml]"' + --command 'gym eval prepare --benchmark wmt24pp' ``` This populates `benchmarks/wmt24pp/data/wmt24pp_benchmark.jsonl` and diff --git a/benchmarks/xstest/README.md b/benchmarks/xstest/README.md index 0e8372cc26..a1bd1fdcfb 100644 --- a/benchmarks/xstest/README.md +++ b/benchmarks/xstest/README.md @@ -3,22 +3,20 @@ Running this benchmark requires 1 GPU for 7B https://huggingface.co/allenai/wild # Prepare data ```bash -config_paths="benchmarks/xstest/config.yaml" -ng_prepare_benchmark "+config_paths=[$config_paths]" +gym eval prepare --benchmark xstest ``` # Run ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/xstest/config.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++output_jsonl_fpath=results/benchmarks/xstest.jsonl \ +gym eval run \ + --model-type vllm_model \ + --benchmark xstest \ + --output results/benchmarks/xstest.jsonl \ + --split benchmark \ + --model-url <> \ + --model-api-key <> \ + --model <> \ + --resume \ ++overwrite_metrics_conflicts=true \ - ++split=benchmark \ - ++resume_from_cache=true \ - ++reuse_existing_data_preparation=true \ - ++policy_base_url=<> \ - ++policy_api_key=<> \ - ++policy_model_name=<> + ++reuse_existing_data_preparation=true ``` diff --git a/environments/workplace_assistant/README.md b/environments/workplace_assistant/README.md index a714aa5f4b..d16862a74c 100644 --- a/environments/workplace_assistant/README.md +++ b/environments/workplace_assistant/README.md @@ -11,17 +11,18 @@ Commands - Spin up server: ``` -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -environments/workplace_assistant/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --config environments/workplace_assistant/config.yaml ``` Collect trajectories: ``` -ng_collect_rollouts +agent_name=workplace_assistant_simple_agent \ - +input_jsonl_fpath=environments/workplace_assistant/data/example.jsonl \ - +output_jsonl_fpath=results/workplace_assistant_trajectory_collection.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent workplace_assistant_simple_agent \ + --input environments/workplace_assistant/data/example.jsonl \ + --output results/workplace_assistant_trajectory_collection.jsonl \ + --limit 1 ``` ## Generating Additional Training Data diff --git a/fern/versions/latest/pages/contribute/development-setup.mdx b/fern/versions/latest/pages/contribute/development-setup.mdx index a9f0e6fbb7..9ca07cf367 100644 --- a/fern/versions/latest/pages/contribute/development-setup.mdx +++ b/fern/versions/latest/pages/contribute/development-setup.mdx @@ -25,9 +25,9 @@ pre-commit install **Run NeMo Gym Tests**: ```bash -ng_dev_test # Run all NeMo Gym core tests -ng_test_all # Run all server tests -ng_test +entrypoint=responses_api_agents/simple_agent # Test single server +gym dev test # Run all NeMo Gym core tests +gym env test # Run all server tests +gym env test --resource-server example_single_tool_call # Test a single resources server ``` **View Test Coverage**: @@ -37,7 +37,7 @@ coverage html # Generate HTML coverage report **Configuration Debugging**: ```bash -ng_dump_config # Dump config as NeMo Gym sees it +gym env resolve # Dump config as NeMo Gym sees it ``` --- @@ -55,7 +55,7 @@ All contributions must pass these automated checks: **Test Requirements**: - At least one test per server you contribute -- Tests must run using `ng_test +entrypoint=your_server_path` +- Tests must run using `gym env test --resource-server your_server` - Use pytest for async testing patterns ### Build Docs CI Failures @@ -244,7 +244,7 @@ pre-commit autoupdate # Update hook versions 1. **Create feature branch**: `git checkout -b feature/your-feature` 2. **Make changes** with tests -3. **Run local checks**: `ng_dev_test && pre-commit run --all-files` +3. **Run local checks**: `gym dev test && pre-commit run --all-files` 4. **Commit with signoff**: `git commit -s -S -m "Your message"` 5. **Push and create PR**: Ensure all CI checks pass 6. **Address review feedback** and iterate diff --git a/fern/versions/latest/pages/contribute/environments/new-environment.mdx b/fern/versions/latest/pages/contribute/environments/new-environment.mdx index e2f4af4db9..728f700dad 100644 --- a/fern/versions/latest/pages/contribute/environments/new-environment.mdx +++ b/fern/versions/latest/pages/contribute/environments/new-environment.mdx @@ -63,7 +63,7 @@ Prepare the dataset for your environment: Build your resources server: -- Run `ng_init_resources_server +entrypoint=resources_servers/my_server` to scaffold the new resources server +- Run `gym env init --resource-server my_server` to scaffold the new resources server - Follow the [Single Step Environment](/environment-tutorials/single-step-environment) guide to implement your specific logic - Implement verification logic for your tasks by defining the `verify()` function - Set the `domain` field in your resources server configuration (see `Domain`). @@ -80,7 +80,7 @@ Write and run tests for your resources server: Verify basic functionality and generate example rollouts: -- Document the command used to start your server, for example, `ng_run +entrypoint=resources_servers/my_server` +- Document the command used to start your server, for example, `gym env run --resource-server my_server` - Generate rollouts and save 5 example outputs to `data/example_rollouts.jsonl` to demonstrate correct reward signals ### 5. Reward Profiling diff --git a/fern/versions/latest/pages/data/download-huggingface.mdx b/fern/versions/latest/pages/data/download-huggingface.mdx index b49cb97b51..d1f237390f 100644 --- a/fern/versions/latest/pages/data/download-huggingface.mdx +++ b/fern/versions/latest/pages/data/download-huggingface.mdx @@ -14,10 +14,10 @@ Download JSONL datasets from Hugging Face Hub for NeMo Gym training. ## Quick Start ```bash -ng_download_dataset_from_hf \ - +repo_id=nvidia/Nemotron-RL-math-OpenMathReasoning \ - +split=train \ - +output_fpath=./data/train.jsonl +gym dataset download \ + --repo-id nvidia/Nemotron-RL-math-OpenMathReasoning \ + --split train \ + --output ./data/train.jsonl ``` ```text @@ -55,9 +55,9 @@ Downloads using the `datasets` library and converts to JSONL. **All splits**: ```bash -ng_download_dataset_from_hf \ - +repo_id=nvidia/Nemotron-RL-knowledge-mcqa \ - +output_dirpath=./data/ +gym dataset download \ + --repo-id nvidia/Nemotron-RL-knowledge-mcqa \ + --output-dir ./data/ ``` ```text @@ -68,10 +68,10 @@ ng_download_dataset_from_hf \ **Single split**: ```bash -ng_download_dataset_from_hf \ - +repo_id=SWE-Gym/SWE-Gym \ - +split=train \ - +output_fpath=./data/train.jsonl +gym dataset download \ + --repo-id SWE-Gym/SWE-Gym \ + --split train \ + --output ./data/train.jsonl ``` @@ -82,10 +82,10 @@ Downloads a specific file directly without conversion. **Use when**: Repository contains pre-formatted JSONL files. ```bash -ng_download_dataset_from_hf \ - +repo_id=nvidia/nemotron-RL-coding-competitive_coding \ - +artifact_fpath=opencodereasoning_filtered_25k_train.jsonl \ - +output_fpath=./data/train.jsonl +gym dataset download \ + --repo-id nvidia/nemotron-RL-coding-competitive_coding \ + --artifact opencodereasoning_filtered_25k_train.jsonl \ + --output ./data/train.jsonl ``` ```text @@ -208,9 +208,9 @@ Avoid passing tokens on the command line—they appear in shell history. ```bash export HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxxxxxxx -ng_download_dataset_from_hf \ - +repo_id=my-org/private-dataset \ - +output_dirpath=./data/ +gym dataset download \ + --repo-id my-org/private-dataset \ + --output-dir ./data/ ``` Get your token at [huggingface.co/settings/tokens](https://huggingface.co/settings/tokens). Use a **read-only** token. @@ -220,10 +220,10 @@ Get your token at [huggingface.co/settings/tokens](https://huggingface.co/settin Not recommended for shared systems: ```bash -ng_download_dataset_from_hf \ - +repo_id=my-org/private-dataset \ - +hf_token=hf_xxxxxxxxxxxxxxxxxxxxxxxxx \ - +output_dirpath=./data/ +gym dataset download \ + --repo-id my-org/private-dataset \ + --output-dir ./data/ \ + +hf_token=hf_xxxxxxxxxxxxxxxxxxxxxxxxx ``` @@ -245,11 +245,11 @@ datasets: Run with download enabled: ```bash -config_paths="resources_servers/code_gen/configs/code_gen.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=./data/prepared \ - +mode=train_preparation \ - +should_download=true \ +gym dataset collate \ + --resource-server code_gen \ + --output-dir ./data/prepared \ + --mode train_preparation \ + --download \ +data_source=huggingface ``` @@ -286,7 +286,7 @@ rm -rf ~/.cache/huggingface/hub/datasets---- -Preprocess raw data, run `ng_prepare_data`, and add `agent_ref` routing. +Preprocess raw data, run `gym dataset collate`, and add `agent_ref` routing. diff --git a/fern/versions/latest/pages/data/index.mdx b/fern/versions/latest/pages/data/index.mdx index 0fe7f47b0e..c8dc02d465 100644 --- a/fern/versions/latest/pages/data/index.mdx +++ b/fern/versions/latest/pages/data/index.mdx @@ -34,7 +34,7 @@ Additional fields like `expected_answer` vary by resources server—the componen | Field | Added By | Description | |-------|----------|-------------| | `responses_create_params` | User | Input to the model during training. Contains `input` (messages) and optional `tools`, `temperature`, etc. | -| `agent_ref` | `ng_prepare_data` | Routes each row to its agent server. Auto-generated during data preparation. | +| `agent_ref` | `gym dataset collate` | Routes each row to its agent server. Auto-generated during data preparation. | ### Optional Fields @@ -60,7 +60,7 @@ The `agent_ref` field maps each row to a specific agent server, which in turn kn } ``` -**You don't create `agent_ref` manually.** The `ng_prepare_data` tool adds it automatically based on your config file. The tool matches the agent type (`responses_api_agents`) with the agent name from the config. +**You don't create `agent_ref` manually.** The `gym dataset collate` tool adds it automatically based on your config file. The tool matches the agent type (`responses_api_agents`) with the agent name from the config. ### Example Data @@ -75,12 +75,11 @@ The `agent_ref` field maps each row to a specific agent server, which in turn kn Run this command from the repository root: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/example_multi_step/configs/example_multi_step.yaml" - -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/test \ - +mode=example_validation +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resource-server example_multi_step \ + --output-dir data/test \ + --mode example_validation ``` **Success**: `Finished!` message and `data/test/example_metrics.json` created. @@ -126,7 +125,7 @@ datasets: ```mermaid flowchart LR A[Create JSONL] --> B[Add to config] - B --> C[Run ng_prepare_data] + B --> C[Run gym dataset collate] C -->|Pass| D[Train with NeMo RL] C -->|Fail| E[Fix and retry] ``` @@ -141,13 +140,12 @@ flowchart LR To prepare training data with auto-download: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/workplace_assistant/configs/workplace_assistant.yaml" - -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/workplace_assistant \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resource-server workplace_assistant \ + --output-dir data/workplace_assistant \ + --mode train_preparation \ + --download ``` @@ -162,7 +160,7 @@ HuggingFace downloads require authentication. Set `hf_token` in `env.yaml` or ex | `JSON parse error at line N` | Invalid JSON | Check quotes, commas, brackets at line N | | `ValidationError: responses_create_params` | Missing field | Add `responses_create_params.input` | | `A license is required` | Missing license | Add `license` to dataset config | -| `Missing local datasets` | File not found | Check path or add `+should_download=true` | +| `Missing local datasets` | File not found | Check path or add `--download` | ## Guides @@ -192,8 +190,8 @@ YAML-based prompt templates applied at rollout time. | Command | Description | |---------|-------------| -| `ng_prepare_data` | Validate and generate metrics | -| `ng_download_dataset_from_hf` | Download from HuggingFace | +| `gym dataset collate` | Validate and generate metrics | +| `gym dataset download` | Download from HuggingFace | See [CLI Commands](/reference/cli-commands) for details. diff --git a/fern/versions/latest/pages/data/prepare-validate.mdx b/fern/versions/latest/pages/data/prepare-validate.mdx index 3c024e6e95..503a69344b 100644 --- a/fern/versions/latest/pages/data/prepare-validate.mdx +++ b/fern/versions/latest/pages/data/prepare-validate.mdx @@ -3,7 +3,7 @@ title: "Prepare and Validate" description: "" position: 2 --- -Format and validate JSONL datasets for NeMo Gym training using `ng_prepare_data`. +Format and validate JSONL datasets for NeMo Gym training using `gym dataset collate`. @@ -13,7 +13,7 @@ Format and validate JSONL datasets for NeMo Gym training using `ng_prepare_data` **In this guide, you will**: -1. Validate datasets with `ng_prepare_data` +1. Validate datasets with `gym dataset collate` 2. Generate training and validation splits 3. Understand the JSONL data format @@ -30,12 +30,11 @@ Format and validate JSONL datasets for NeMo Gym training using `ng_prepare_data` From the repository root: ```bash -config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_prepare_data \ - "+config_paths=[$config_paths]" \ - +output_dirpath=data/test \ - +mode=example_validation +gym dataset collate \ + --resource-server example_multi_step \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/test \ + --mode example_validation ``` Success output: @@ -106,7 +105,7 @@ Check `resources_servers//README.md` for required fields specific to each ## Preprocess Raw Datasets -If your dataset doesn't have `responses_create_params`, you need to preprocess it before using `ng_prepare_data`. +If your dataset doesn't have `responses_create_params`, you need to preprocess it before using `gym dataset collate`. **When to preprocess**: - Downloaded datasets without NeMo Gym format @@ -178,7 +177,7 @@ wc -l train.jsonl validation.jsonl ### Create Config for Custom Data -After preprocessing, create a config file to point `ng_prepare_data` at your local files. +After preprocessing, create a config file to point `gym dataset collate` at your local files. @@ -216,8 +215,11 @@ custom_simple_agent: Run data preparation: ```bash -config_paths="custom_data.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" +mode=train_preparation +output_dirpath=data +gym dataset collate \ + --config custom_data.yaml \ + --config responses_api_models/vllm_model/configs/vllm_model.yaml \ + --mode train_preparation \ + --output-dir data ``` This validates your data and adds the `agent_ref` field to each row, routing samples to your resources server. @@ -234,28 +236,32 @@ This validates your data and adds the `agent_ref` field to each row, routing sam ### Example Validation ```bash -ng_prepare_data "+config_paths=[resources_servers/example_multi_step/configs/example_multi_step.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +output_dirpath=data/example_multi_step \ - +mode=example_validation +gym dataset collate \ + --resource-server example_multi_step \ + --config responses_api_models/vllm_model/configs/vllm_model.yaml \ + --output-dir data/example_multi_step \ + --mode example_validation ``` ### Training Preparation ```bash -ng_prepare_data "+config_paths=[resources_servers/workplace_assistant/configs/workplace_assistant.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +output_dirpath=data/workplace_assistant \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --resource-server workplace_assistant \ + --config responses_api_models/vllm_model/configs/vllm_model.yaml \ + --output-dir data/workplace_assistant \ + --mode train_preparation \ + --download ``` ### CLI Parameters | Parameter | Required | Description | |-----------|----------|-------------| -| `+config_paths` | Yes | YAML config paths | -| `+output_dirpath` | Yes | Output directory | -| `+mode` | Yes | `example_validation` or `train_preparation` | -| `+should_download` | No | Download missing datasets (default: `false`) | +| `--config` | Yes | YAML config paths (repeatable) | +| `--output-dir` | Yes | Output directory | +| `--mode` | Yes | `example_validation` or `train_preparation` | +| `--download` | No | Download missing datasets (default: `false`) | | `+data_source` | No | `huggingface` (default) or `gitlab` | --- @@ -267,7 +273,7 @@ ng_prepare_data "+config_paths=[resources_servers/workplace_assistant/configs/wo | Missing `responses_create_params` | Sample silently skipped | Add field with valid `input` | | Invalid JSON | Sample skipped | Fix JSON syntax | | Invalid role | Sample skipped | Use `user`, `assistant`, `system`, or `developer` | -| Missing dataset file | `AssertionError` | Create file or set `+should_download=true` | +| Missing dataset file | `AssertionError` | Create file or pass `--download` | Invalid samples are silently skipped. If metrics show fewer examples than expected, check your data format. @@ -305,7 +311,7 @@ with open("your_data.jsonl") as f: ## Validation Process -`ng_prepare_data` performs these steps: +`gym dataset collate` performs these steps: 1. **Load configs** — Parse server configs, identify datasets 2. **Check files** — Verify dataset files exist diff --git a/fern/versions/latest/pages/data/prompt-config.mdx b/fern/versions/latest/pages/data/prompt-config.mdx index ed304f656e..bb998a72a8 100644 --- a/fern/versions/latest/pages/data/prompt-config.mdx +++ b/fern/versions/latest/pages/data/prompt-config.mdx @@ -14,8 +14,8 @@ Apply YAML-based prompt templates at rollout time to build `responses_create_par **In this guide, you will**: 1. Write a prompt config YAML file -2. Apply it during rollout collection with `ng_collect_rollouts` -3. Optionally materialize prompts into JSONL with `ng_materialize_prompts` +2. Apply it during rollout collection with `gym eval run --no-serve` +3. Optionally materialize prompts into JSONL with `gym dataset render` @@ -73,31 +73,32 @@ Literal braces must be doubled (`{{` / `}}`). For example, `\\boxed{{}}` produce ### At rollout time -Pass `+prompt_config=` to `ng_collect_rollouts`: +Pass `--prompt-config ` to `gym eval run --no-serve`: ```bash -ng_collect_rollouts \ - +agent_name=my_agent \ - +input_jsonl_fpath=data/raw_problems.jsonl \ - +output_jsonl_fpath=results/rollouts.jsonl \ - +prompt_config=/path/to/my_prompt.yaml \ - +num_repeats=5 \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent my_agent \ + --input data/raw_problems.jsonl \ + --output results/rollouts.jsonl \ + --prompt-config /path/to/my_prompt.yaml \ + --num-repeats 5 \ + --max-output-tokens 16384 \ + --temperature 1.0 ``` -The `+prompt_config` path must be either an absolute path or a path relative to the Gym repository root. +The `--prompt-config` path must be either an absolute path or a path relative to the Gym repository root. The input JSONL should contain raw fields (e.g. `question`, `expected_answer`) **without** `responses_create_params.input`. The prompt config builds the input messages during rollout collection. ### Standalone materialization -Use `ng_materialize_prompts` to write a prompt template into JSONL without running rollouts: +Use `gym dataset render` to write a prompt template into JSONL without running rollouts: ```bash -ng_materialize_prompts \ - +input_jsonl_fpath=data/raw_problems.jsonl \ - +prompt_config=/path/to/my_prompt.yaml \ - +output_jsonl_fpath=data/materialized.jsonl +gym dataset render \ + --input data/raw_problems.jsonl \ + --prompt-config /path/to/my_prompt.yaml \ + --output data/materialized.jsonl ``` This produces a new JSONL file with `responses_create_params.input` populated from the template. This is useful for inspection or passing to other tools that expect pre-populated input. @@ -119,21 +120,21 @@ Other fields in `responses_create_params` (such as `tools` and `temperature`) ar ## CLI Parameters -### `ng_collect_rollouts` +### `gym eval run --no-serve` | Parameter | Required | Description | |-----------|----------|-------------| -| `+prompt_config` | No | Path to a prompt YAML file. Mutually exclusive with pre-populated `responses_create_params.input` in the JSONL data. | +| `--prompt-config` | No | Path to a prompt YAML file. Mutually exclusive with pre-populated `responses_create_params.input` in the JSONL data. | -See [CLI Commands](/reference/cli-commands) for the full list of `ng_collect_rollouts` parameters. +See [CLI Commands](/reference/cli-commands) for the full list of `gym eval run --no-serve` parameters. -### `ng_materialize_prompts` +### `gym dataset render` | Parameter | Required | Description | |-----------|----------|-------------| -| `+input_jsonl_fpath` | Yes | Raw JSONL data (no `responses_create_params.input`). | -| `+prompt_config` | Yes | Path to prompt YAML file to apply. | -| `+output_jsonl_fpath` | Yes | Output path for materialized JSONL with populated prompts. | +| `--input` | Yes | Raw JSONL data (no `responses_create_params.input`). | +| `--prompt-config` | Yes | Path to prompt YAML file to apply. | +| `--output` | Yes | Output path for materialized JSONL with populated prompts. | --- diff --git a/fern/versions/latest/pages/environment-tutorials/aggregate-metrics.mdx b/fern/versions/latest/pages/environment-tutorials/aggregate-metrics.mdx index f4da3e8793..193c6fab5d 100644 --- a/fern/versions/latest/pages/environment-tutorials/aggregate-metrics.mdx +++ b/fern/versions/latest/pages/environment-tutorials/aggregate-metrics.mdx @@ -9,7 +9,7 @@ After rollout collection, NeMo Gym computes **aggregate metrics** for each agent ## How It Works -1. **Rollouts complete** — `ng_collect_rollouts` gathers verify responses (reward + custom fields) for every task/rollout pair. +1. **Rollouts complete** — `gym eval run --no-serve` gathers verify responses (reward + custom fields) for every task/rollout pair. 2. **Group by agent** — responses are partitioned by agent name. 3. **Call `/aggregate_metrics`** — for each agent, the stripped verify responses are POSTed to the agent's `/aggregate_metrics` endpoint. 4. **Compute stats** — per-task and overall statistics (`mean`, `max`, `min`, `median`, `std`) are computed for every numeric field. If the resources server overrides `compute_metrics()` or `get_key_metrics()`, those are called to add additional metrics. diff --git a/fern/versions/latest/pages/environment-tutorials/integrate-external-environments.mdx b/fern/versions/latest/pages/environment-tutorials/integrate-external-environments.mdx index 514c209347..2362d620b2 100644 --- a/fern/versions/latest/pages/environment-tutorials/integrate-external-environments.mdx +++ b/fern/versions/latest/pages/environment-tutorials/integrate-external-environments.mdx @@ -12,7 +12,7 @@ Integrating external training environments, benchmarks, or agents into a NeMo Gy 1. Because external training environments, benchmarks, or agents include their own rollout orchestration logic (that is, coordinating model and tool calls) and may sometimes return their own reward, NeMo Gym agent servers are the appropriate level to integrate at. 2. You can add a dependency from your NeMo Gym agent server to the third-party library by adding it to the requirements.txt. If your dependency needs are more complicated beyond installing pip packages or GitHub repositories, consider using setup.py and pyproject.toml. 3. After you add a dependency, you can import it in app.py like a normal Python script. -4. You can run the `ng_test` command to run tests on your agent server. Refer to the [ng_test CLI reference](/reference/cli-commands#ng-test-nemo-gym-test) for details. +4. You can run the `gym env test` command to run tests on your agent server. Refer to the [gym env test CLI reference](/reference/cli-commands#gym-env-test) for details. 5. Typically you can just wrap the external library in the `/run` function and omit the `/responses` function. 6. In the `/run` function before calling into the third-party library, you will need to preprocess from NeMo Gym schema into a config for the external library. 7. In the `/run` function after calling into the third-party library, you will need to postprocess from the external library result into a NeMo Gym response. diff --git a/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx b/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx index 743b3a6abe..f635f028a5 100644 --- a/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx +++ b/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx @@ -53,7 +53,7 @@ In most cases, the **Resources Server** is where your changes go: define your to Resource servers live in the `resources_servers/` directory. Scaffold a weather server that provides weather information to models: ```bash -ng_init_resources_server +entrypoint=resources_servers/my_weather_tool +gym env init --resource-server my_weather_tool ``` This generates the following structure along with a paired simple agent configuration: @@ -393,7 +393,7 @@ async def test_verify_without_tool_call(server): Run the tests: ```bash -ng_test +entrypoint=resources_servers/my_weather_tool +gym env test --resource-server my_weather_tool ``` For detailed test output: @@ -413,13 +413,12 @@ pytest -v Start the servers: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/my_weather_tool/configs/my_weather_tool.yaml" - -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --resource-server my_weather_tool ``` -`ng_run` reads the config files and starts all three components from the architecture diagram: +`gym env run` reads the config files and starts all three components from the architecture diagram: 1. **Agent Server** (`my_weather_tool_simple_agent`) — the `simple_agent` that orchestrates the seed → model → tool → verify loop 2. **Model Server** (`openai_model`) — proxies LLM inference requests to the OpenAI API @@ -475,12 +474,13 @@ Before training, you collect rollouts to validate that your environment works en With your servers still running, collect rollouts against your example inputs: ```bash -ng_collect_rollouts +agent_name=my_weather_tool_simple_agent \ - +input_jsonl_fpath=resources_servers/my_weather_tool/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/my_weather_tool/data/example_rollouts.jsonl \ - +limit=null \ - +num_repeats=null \ - +num_samples_in_parallel=null +gym eval run --no-serve \ + --agent my_weather_tool_simple_agent \ + --input resources_servers/my_weather_tool/data/example.jsonl \ + --output resources_servers/my_weather_tool/data/example_rollouts.jsonl \ + --limit null \ + --num-repeats null \ + --concurrency null ``` @@ -545,7 +545,7 @@ We'd love to see your contributions! Please make sure your PR includes accurate You've learned how to: -- Initialize a resource server with `ng_init_resources_server` +- Initialize a resource server with `gym env init` - Prepare task data in JSONL format - Implement tool endpoints and verification logic - Configure the required `domain` field and wire components together @@ -654,7 +654,7 @@ Check server status and logs: ```bash # View running servers -ng_status +gym env status # For detailed logs, run the server directly: cd resources_servers/my_weather_tool @@ -662,4 +662,4 @@ source .venv/bin/activate python app.py ``` -Server logs appear in the terminal where `ng_run` was executed. +Server logs appear in the terminal where `gym env run` was executed. diff --git a/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx b/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx index 8843c5109a..77417f09b2 100644 --- a/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx +++ b/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx @@ -94,7 +94,7 @@ The resources server config points the judge at the policy model — `judge_mode The config file ships with a `judge_model` block that starts a dedicated judge server. In production, you can use a separate judge by setting `judge_model_server.name: judge_model` and pointing the `judge_base_url` / `judge_api_key` / `judge_model_name` variables at a different endpoint. This lets you use a different model, provider, or quota for the judge. -Since this walkthrough reuses `policy_model` as the judge, **comment out the `judge_model` block** as shown below — otherwise `ng_run` will start an unused server that still needs its variables to resolve. +Since this walkthrough reuses `policy_model` as the judge, **comment out the `judge_model` block** as shown below — otherwise `gym env run` will start an unused server that still needs its variables to resolve. Be sure to **set `judge_model_server.name` to `policy_model`** as well. @@ -200,18 +200,21 @@ If you are building your own LLM-judge server, you will write similar code — t Start the servers: ```bash -ng_run "+config_paths=[resources_servers/over_refusal_detection/configs/over_refusal_detection.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env run \ + --resource-server over_refusal_detection \ + --model-type openai_model ``` In another terminal, collect rollouts against the 5-entry example dataset to confirm the judge call and reward parsing work end-to-end: ```bash -ng_collect_rollouts \ - +agent_name=over_refusal_detection_simple_agent \ - +input_jsonl_fpath=resources_servers/over_refusal_detection/data/example.jsonl \ - +output_jsonl_fpath=/tmp/over_refusal_smoke_test.jsonl \ - +num_repeats=1 \ - "+responses_create_params={max_output_tokens: 1024, temperature: 1.0}" +gym eval run --no-serve \ + --agent over_refusal_detection_simple_agent \ + --input resources_servers/over_refusal_detection/data/example.jsonl \ + --output /tmp/over_refusal_smoke_test.jsonl \ + --num-repeats 1 \ + --max-output-tokens 1024 \ + --temperature 1.0 ``` Inspect the output JSONL to verify that `reward` values are `0.0`, `0.5`, or `1.0` as expected. Once this looks right, scale to larger datasets and higher `num_repeats`. @@ -347,7 +350,7 @@ Other servers apply the same pattern with domain-specific variations. For exampl 2. Add or reuse a **model server** for the judge; reference it from `judge_model_server`. 3. Design **prompts and parseable verdicts**; handle judge failures gracefully. 4. Set **temperature / max tokens** and **concurrency** for your SLA and budget. -5. Smoke-test with `ng_run` and your resources server's **`data/example.jsonl`**, then scale with `ng_collect_rollouts`. +5. Smoke-test with `gym env run` and your resources server's **`data/example.jsonl`**, then scale with `gym eval run --no-serve`. Done looks like: diff --git a/fern/versions/latest/pages/get-started/installation.mdx b/fern/versions/latest/pages/get-started/installation.mdx index bd20b4f646..ff3b5281be 100644 --- a/fern/versions/latest/pages/get-started/installation.mdx +++ b/fern/versions/latest/pages/get-started/installation.mdx @@ -33,7 +33,7 @@ docker pull nvcr.io/nvidia/nemo-rl:v0.5.0.nemotron_3_super ## Verify Installation ```bash -ng_version +gym --version ``` You should see output like: diff --git a/fern/versions/latest/pages/get-started/quickstart.mdx b/fern/versions/latest/pages/get-started/quickstart.mdx index ebcbea311f..7eec1d9929 100644 --- a/fern/versions/latest/pages/get-started/quickstart.mdx +++ b/fern/versions/latest/pages/get-started/quickstart.mdx @@ -31,10 +31,9 @@ Run your agent on a set of tasks and score the results. This example uses a simp NeMo Gym uses local servers to coordinate your model, agent, and task verification. Start them first: ```bash -environment_config="resources_servers/mcqa/configs/mcqa.yaml" -model_config="responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[${environment_config},${model_config}]" +gym env run \ + --resource-server mcqa \ + --model-type openai_model ``` You should see three server instances starting: @@ -52,12 +51,12 @@ In a new terminal, run your agent on a single task to verify everything works: ```bash source .venv/bin/activate -ng_collect_rollouts \ - +agent_name=mcqa_simple_agent \ - +input_jsonl_fpath=resources_servers/mcqa/data/example.jsonl \ - +output_jsonl_fpath=results/mcqa_rollouts.jsonl \ - +limit=5 \ - +num_repeats=1 +gym eval run --no-serve \ + --agent mcqa_simple_agent \ + --input resources_servers/mcqa/data/example.jsonl \ + --output results/mcqa_rollouts.jsonl \ + --limit 5 \ + --num-repeats 1 ``` You should see a progress bar followed by aggregate metrics: @@ -77,7 +76,7 @@ Rollouts: results/mcqa_rollouts.jsonl Aggregate metrics: results/mcqa_rollouts_aggregate_metrics.json ``` -For per-task pass rates, see `ng_reward_profile` in the [CLI Reference](/reference/cli-commands). +For per-task pass rates, see `gym eval profile` in the [CLI Reference](/reference/cli-commands). ## Explore @@ -86,7 +85,7 @@ Now that you have a working setup, explore what's available. NeMo Gym ships with environments across many domains. You can use these existing environments in addition to building your own. ```bash -ng_list_benchmarks +gym list benchmarks ``` ```text @@ -107,11 +106,11 @@ Available benchmarks in NeMo Gym This lists benchmarks with pre-configured agents. For the full set of environments (including training environments), see the [Available Environments](https://github.com/NVIDIA-NeMo/Gym#-available-environments) table. -Every CLI command supports `+h=true` or `+help=true` for detailed usage information: +Every CLI command supports `-h` or `--help` for detailed usage information: ```bash -ng_help -ng_run +help=true +gym --help +gym env run --help ``` ## Next Steps diff --git a/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx b/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx index f467c1316c..017101aa6f 100644 --- a/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx +++ b/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx @@ -6,7 +6,7 @@ This section describes how NeMo Gym components interact during startup and execu ## Control Plane: Server Startup -When you run `ng_run`, the system starts up in four phases: +When you run `gym env run`, the system starts up in four phases: ```mermaid %%{init: {'theme': 'default', 'themeVariables': { 'lineColor': '#5c6bc0', 'primaryTextColor': '#333', 'primaryColor': '#e3f2fd', 'secondaryColor': '#f5f5f5'}}}%% @@ -18,10 +18,12 @@ flowchart LR ### Phase 1: Parse CLI -The `ng_run` command uses Hydra to parse command-line arguments. Users specify configuration files via `+config_paths`: +The `gym env run` command uses Hydra to parse command-line arguments. Users specify configuration files via `--config`: ```bash -ng_run "+config_paths=[resources_servers/math/configs/math.yaml, responses_api_models/openai_model/configs/openai_model.yaml]" +gym env run \ + --resource-server math \ + --model-type openai_model ``` ### Phase 2: Load and Merge Configs @@ -45,7 +47,7 @@ Servers are started in two stages: ```mermaid %%{init: {'theme': 'default', 'themeVariables': { 'lineColor': '#5c6bc0', 'primaryTextColor': '#333'}}}%% flowchart TB - Main["Main Process
ng_run"] + Main["Main Process
gym env run"] Head["Head Server
Port 11000"] Model["Model Server
Own venv + port"] Agent["Agent Server
Own venv + port"] @@ -133,7 +135,7 @@ sequenceDiagram ## Data Plane: Rollout Collection -When you run `ng_collect_rollouts`, the system collects training data by executing rollouts in parallel: +When you run `gym eval run --no-serve`, the system collects training data by executing rollouts in parallel: ```mermaid %%{init: {'theme': 'default', 'themeVariables': { 'lineColor': '#5c6bc0', 'primaryTextColor': '#333'}}}%% @@ -174,5 +176,5 @@ sequenceDiagram The client first queries the Head Server to discover server addresses from the global config, then reads input JSONL and dispatches prompts to the Agent. Completed rollouts are written to output JSONL. **Concurrency behavior differs by use case:** -- **Standalone rollout collection** (`ng_collect_rollouts`): A semaphore gates concurrency via `num_samples_in_parallel` to control load. +- **Standalone rollout collection** (`gym eval run --no-serve`): A semaphore gates concurrency via `num_samples_in_parallel` to control load. - **Training framework integration** (e.g., NeMo RL): All requests are sent without gating; the training framework manages concurrency externally. diff --git a/fern/versions/latest/pages/model-server/vllm.mdx b/fern/versions/latest/pages/model-server/vllm.mdx index 6391e1d109..68418610b9 100644 --- a/fern/versions/latest/pages/model-server/vllm.mdx +++ b/fern/versions/latest/pages/model-server/vllm.mdx @@ -11,9 +11,9 @@ VLLMModel provides a Responses API to Chat Completions mapping middleware layer **To use VLLMModel, just change the `responses_api_models/openai_model/configs/openai_model.yaml` in your config paths to `responses_api_models/vllm_model/configs/vllm_model.yaml`!** ```bash -config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --resource-server example_multi_step \ + --model-type vllm_model ``` ## Use VLLMModel @@ -110,12 +110,12 @@ vllm serve \ ### Configure NeMo Gym to use the local vLLM server In a second terminal on the same GPU node that was used to spin up the vLLM server, enter the NeMo Gym Python environment, and start the NeMo Gym servers. ```bash -config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[$config_paths]" \ - ++policy_base_url=http://0.0.0.0:10240/v1 \ - ++policy_model_name=Qwen/Qwen3-4B-Thinking-2507 \ - ++policy_api_key=dummy_key +gym env run \ + --resource-server example_multi_step \ + --model-type vllm_model \ + --model-url http://0.0.0.0:10240/v1 \ + --model Qwen/Qwen3-4B-Thinking-2507 \ + --model-api-key dummy_key ``` @@ -131,9 +131,10 @@ Then replace the `policy_base_url=http://0.0.0.0:10240/v1` to point to the hostn ### Run rollout collection In a third terminal on the same GPU node that was used to spin up the vLLM server, enter the NeMo Gym Python environment, and run rollout collection. ```bash -ng_collect_rollouts +agent_name=example_multi_step_simple_agent \ - +input_jsonl_fpath=resources_servers/example_multi_step/data/example.jsonl \ - +output_jsonl_fpath=results/example_multi_step_rollouts.jsonl +gym eval run --no-serve \ + --agent example_multi_step_simple_agent \ + --input resources_servers/example_multi_step/data/example.jsonl \ + --output results/example_multi_step_rollouts.jsonl ``` ## VLLMModel configuration reference diff --git a/fern/versions/latest/pages/reference/cli-commands.mdx b/fern/versions/latest/pages/reference/cli-commands.mdx index d24b69612a..61851029fd 100644 --- a/fern/versions/latest/pages/reference/cli-commands.mdx +++ b/fern/versions/latest/pages/reference/cli-commands.mdx @@ -1,554 +1,630 @@ --- title: "CLI Commands" -description: "" +description: "Reference for the unified gym CLI: command groups, flags, and migration from the legacy ng_* commands." position: 3 --- -This page documents all available NeMo Gym CLI commands. +This page documents the NeMo Gym command-line interface. - -Each command has both a short form (such as `ng_run`) and a full form (such as `nemo_gym_run`). They are functionally identical. +All functionality is exposed through a single `gym` entry point, organized into command groups (`gym `). `ng` is a drop-in alias for `gym`. Every group and command supports `-h`/`--help`. + +The legacy `ng_*` / `nemo_gym_*` commands (such as `ng_run` or `nemo_gym_collect_rollouts`) still work but are deprecated. Each one prints a notice pointing at its `gym` replacement and then runs it. See [Migrating from the legacy commands](#migrating-from-the-legacy-commands) at the bottom of this page. ## Quick Reference ```bash -# Display help -ng_help - -# Get detailed help for any command -ng_run +help=true -ng_test +h=true +# General +gym --help # list all command groups +gym --version [--json] # print version and system info +ng ... # 'ng' is an alias for 'gym' + +# Discover benchmarks +gym list benchmarks [--json] # list available benchmarks +gym search [--json] # filter the benchmark list by name + +# Datasets +gym dataset upload # upload a prepared dataset to HF (default) or GitLab +gym dataset download # download a dataset from HF (default) or GitLab +gym dataset rm # delete a dataset from GitLab +gym dataset migrate # move a dataset from GitLab to HF +gym dataset render # generate a dataset preview (materialize prompts) +gym dataset collate # validate and collate a dataset + +# Environments +gym env init # scaffold a new resources server +gym env resolve # resolve and print the final merged config +gym env packages # list packages in a server's virtual environment +gym env test # test resource server(s); all of them if none is given +gym env run # start the servers +gym env status # show running servers + +# Evaluation +gym eval prepare # prepare benchmark data and dump it to disk +gym eval run # collate data, start servers, and collect rollouts +gym eval aggregate # merge sharded rollout results +gym eval profile # compute a reward profile from rollouts + +# Contributor helpers +gym dev test # run NeMo Gym's unit tests ``` ---- - -## Server Management - -Commands for running, testing, and managing NeMo Gym servers. +## Common Options -### `ng_run` / `nemo_gym_run` +These options are shared across many commands. -Start NeMo Gym servers for agents, models, and resources. +| Option | Description | +| --- | --- | +| `--config PATH` | Load a Gym config YAML. Repeatable. Maps to `+config_paths=[...]`. | +| `--benchmark NAME` | Select a registered benchmark by name instead of a config path. | +| `--resource-server NAME` | Select a registered resources server by name. | +| `--model-type NAME` | Select a registered model server type by name (such as `openai_model` or `vllm_model`). | +| `--search-dir DIR` | Extra root directory to search for named components. Repeatable. Lets you register your own benchmarks, resources servers, and models. | +| `--json` | Emit machine-readable JSON instead of human-readable output (reporting commands only). | +| `-v`, `--verbose` | Set the logging level to DEBUG. Flows through to spun-up servers. | +| `-h`, `--help` | Show help for any group or command. | -This command reads configuration from YAML files specified via `+config_paths` and starts all configured servers. The configuration files should define server instances with their entrypoints and settings. + +The `--benchmark`, `--resource-server`, and `--model-type` selectors resolve a component name to its config file for you, so you do not need to know the project's directory layout. If you mistype a name, the CLI suggests the closest match. To point at a config file directly, use `--config ` instead. + -**Configuration Parameter** +### Hydra overrides (escape hatch) -| Parameter | Type | Description | -| --- | --- | --- | -| `config_paths` | List[str] | Paths to YAML configuration files. Specify using Hydra: `+config_paths="[file1.yaml,file2.yaml]"` | +The `gym` CLI is a thin wrapper over Gym's Hydra config system. Standard flags (`--flag value`) cover the common inputs. Anything not covered by a flag can still be passed as a raw Hydra override using `+key=value` (add a new key) or `++key=value` (override an existing key). Unknown overrides are forwarded to Hydra untouched, so advanced config composition keeps working. -**Example** +When a command needs overrides for keys that have no dedicated flag, prefer keeping the whole command in Hydra form (config paths included) rather than mixing `--flag` and `+key=value` styles in one invocation: ```bash -# Start servers with specific configs -config_paths="resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym eval run \ + --benchmark aime24 \ + --model-type openai_model \ + ++responses_create_params.reasoning.effort=low \ + +wandb_project=gym-dev ``` ---- - -### `ng_test` / `nemo_gym_test` - -Test a specific server module by running its pytest suite and optionally validating example data. +Unknown `--flags` (note the leading dashes) are treated as typos and rejected with a "did you mean?" hint, since Hydra overrides never start with `-`. -**Parameters** +### Selecting the model server -| Parameter | Type | Description | -| --- | --- | --- | -| `entrypoint` | str | Entrypoint for this command. Must be a relative path with two parts (such as `responses_api_agents/simple_agent`). | -| `should_validate_data` | bool | Whether to validate the example data (examples, metrics, rollouts, and so on) for this server. Default: `False`. | +Commands that need a model (`gym env run`, `gym eval run`) configure it through four flags: -**Example** - -```bash -ng_test +entrypoint=resources_servers/example_single_tool_call -``` +| Flag | Description | +| --- | --- | +| `--model-type NAME` | The model server type to load (such as `openai_model`, `vllm_model`, or `local_vllm_model`). Derives the correct server implementation. | +| `--model`, `-m` | The served model identifier: an API model name, an HF id, or a local checkpoint path. Interpreted per `--model-type`. Maps to `policy_model_name`. | +| `--model-url` | Base URL of an existing model server endpoint. Maps to `policy_base_url`. | +| `--model-api-key` | API key for the model server. Maps to `policy_api_key`. | --- -### `ng_test_all` / `nemo_gym_test_all` - -Run tests for all server modules in the project. +## General -**Parameters** +### `gym --help` -| Parameter | Type | Description | -| --- | --- | --- | -| `fail_on_total_and_test_mismatch` | bool | Fail if the number of server modules does not match the number with tests. Default: `False`. | - -**Example** +List all command groups. Replaces `ng_help`. ```bash -ng_test_all +gym --help ``` ---- - -### `ng_dev_test` / `nemo_gym_dev_test` +### `gym --version` -Run core NeMo Gym tests with coverage reporting. Runs pytest with the `--cov` flag. +Print the NeMo Gym version along with Python, key dependency, and system information. Replaces `ng_version`. -**Example** +| Option | Description | +| --- | --- | +| `--json` | Output version information as JSON. | ```bash -ng_dev_test -``` - ---- +gym --version -### `ng_init_resources_server` / `nemo_gym_init_resources_server` - -Initialize a new resources server with template files and directory structure. - -**Example** - -```bash -ng_init_resources_server +entrypoint=resources_servers/my_server +# Output as JSON +gym --version --json ``` --- -## Data Collection - -Commands for collecting verified rollouts for RL training. +## Discovery -### `ng_collect_rollouts` / `nemo_gym_collect_rollouts` +Commands for discovering what you can evaluate against. -Perform a batch of rollout collection. +### `gym list benchmarks` -**Parameters** +List the benchmarks available in NeMo Gym, with their domain, agent, and configured number of repeats. -| Parameter | Type | Description | -| --- | --- | --- | -| `agent_name` | str | The agent to collect rollouts from. | -| `input_jsonl_fpath` | str | The input data source to use to collect rollouts, in the form of a file path to a JSONL file. | -| `output_jsonl_fpath` | str | The output data JSONL file path. | -| `limit` | Optional[int] | Maximum number of examples to load and take from the input dataset. | -| `num_repeats` | Optional[int] | The number of times to repeat each example to run. Useful if you want to calculate mean@k, such as mean@4 or mean@16. | -| `num_repeats_add_seed` | bool | When num_repeats >1, add a "seed" parameter on the Responses create params. | -| `num_samples_in_parallel` | Optional[int] | Limit the number of concurrent samples running at once. | -| `responses_create_params` | Dict | Overrides for the `responses_create_params`, such as `temperature` and `max_output_tokens`. | - -**Example** +| Option | Description | +| --- | --- | +| `--json` | Output the benchmark list as JSON. | ```bash -ng_collect_rollouts \ - +agent_name=example_single_tool_call_simple_agent \ - +input_jsonl_fpath=weather_query.jsonl \ - +output_jsonl_fpath=weather_rollouts.jsonl \ - +limit=100 \ - +num_repeats=4 \ - +num_samples_in_parallel=10 -``` - -### `ng_e2e_collect_rollouts` / `nemo_gym_e2e_collect_rollouts` - -Spin up all necessary servers and perform a batch of rollout collection using each dataset inside the provided configs. +gym list benchmarks -**Parameters** - -| Parameter | Type | Description | -| --- | --- | --- | -| `output_jsonl_fpath` | str | The output data JSONL file path. | -| `num_samples_in_parallel` | Optional[int] | Limit the number of concurrent samples running at once. | -| `responses_create_params` | Dict | Overrides for the `responses_create_params`, such as `temperature` and `max_output_tokens`. | +# Machine-readable output for scripting +gym list benchmarks --json | jq '.[].name' +``` -**Examples** +### `gym search` -```bash -ng_e2e_collect_rollouts \ - +output_jsonl_fpath=weather_rollouts.jsonl \ - +num_samples_in_parallel=10 -``` +List benchmarks filtered to those whose name, agent, resources server, dataset, or domain fuzzily matches a query. Equivalent to `gym list benchmarks` narrowed to a query. -```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/math_with_judge/configs/math_with_judge.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++wandb_project= \ - ++wandb_name= \ - ++wandb_dir= \ - ++output_jsonl_fpath=results/test_e2e_rollout_collection/aime24.jsonl \ - ++split=validation -``` +| Argument / Option | Description | +| --- | --- | +| `QUERY` | Substring to match against component names (positional, required). | +| `--json` | Output matches as JSON. | -Example using GPT-OSS 120B remote vLLM endpoint ```bash -experiment_name=rollouts/test_001 -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/math_with_judge/configs/math_with_judge.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - +skip_venv_if_present=true \ - +wandb_project=gym-dev \ - +wandb_name=$(date +%Y%m%d)/$experiment_name \ - ++output_jsonl_fpath=results/$experiment_name.jsonl \ - ++overwrite_metrics_conflicts=true \ - ++split=validation \ - ++policy_model_name=openai/gpt-oss-120b \ - ++policy_api_key=dummy_key \ - ++policy_base_url=http://0.0.0.0:10240/v1 \ - ++responses_create_params.reasoning.effort=low \ - ++responses_create_params.temperature=1.0 \ - ++responses_create_params.top_p=1.0 &> eval_gptoss120b.log & +gym search math +gym search aime --json ``` --- -### `ng_reward_profile` / `nemo_gym_reward_profile` - -Computes statistics on rewards and task difficulty for rollouts collected with `ng_collect_rollouts` with `num_repeats` >1. This outputs a new "reward profiled" dataset, where each task in the dataset has metrics like the average reward, standard deviation, min/max, and pass rate. This is useful in filtering tasks before training for difficulty, variance, or creating a curriculum. +## Datasets -**Parameters** +Commands for preparing, previewing, and managing datasets. By default dataset transfer commands use HuggingFace; pass `--storage gitlab` to target the GitLab Model Registry. -| Parameter | Type | Description | -| --- | --- | --- | -| `input_jsonl_fpath` | str | Path to the original task dataset JSONL file. | -| `rollouts_jsonl_fpath` | str | Path to the rollouts file from `ng_collect_rollouts` (must have been run with `num_repeats` >1). | -| `output_jsonl_fpath` | str | Output file path for the reward profiled dataset. | -| `pass_threshold` | Optional[float] | Reward threshold for computing pass rate. If not specified, pass rate metrics are not included. | +### `gym dataset upload` -**Output Fields** +Upload a prepared local JSONL dataset to HuggingFace (default) or GitLab. Replaces `ng_upload_dataset_to_hf` and `ng_upload_dataset_to_gitlab`. -Each output row contains all original task fields plus: -- `avg_reward`: Average reward across all rollouts -- `std_reward`: Standard deviation of rewards -- `min_reward`: Minimum reward observed -- `max_reward`: Maximum reward observed -- `total_samples`: Number of rollout samples -- `pass_rate`, `pass_rate_total`, `pass_rate_passed`, `pass_threshold`: (Only if `pass_threshold` is specified) - -**Example** +| Option | Description | +| --- | --- | +| `--storage {hf,gitlab}` | Storage backend. Default: `hf`. | +| `--input`, `-i` | Local JSONL file to upload. | +| `--name` | Dataset name. | +| `--revision` | Dataset revision (version). | +| `--split` | Dataset split (HF only). | +| `--create-pr` | Open a pull request instead of committing directly (HF only). | ```bash -ng_reward_profile \ - +input_jsonl_fpath=tasks.jsonl \ - +rollouts_jsonl_fpath=rollouts.jsonl \ - +output_jsonl_fpath=profiled_tasks.jsonl \ - +pass_threshold=1.0 +# Upload to HuggingFace +gym dataset upload \ + --name my_dataset \ + --input data/train.jsonl \ + --revision 0.0.1 + +# Upload to GitLab +gym dataset upload \ + --storage gitlab \ + --name my_dataset \ + --input data/train.jsonl \ + --revision 0.0.1 ``` ---- - -## Data Management - -Commands for preparing and viewing training data. - -### `ng_prepare_data` / `nemo_gym_prepare_data` - -Prepare and validate training data, generating metrics and statistics for datasets. +### `gym dataset download` -**Parameters** +Download a dataset from HuggingFace (default) or GitLab. Replaces `ng_download_dataset_from_hf` and `ng_download_dataset_from_gitlab`. -| Parameter | Type | Description | -| --- | --- | --- | -| `output_dirpath` | str | Directory path where processed datasets and metrics will be saved. | -| `mode` | Literal["train_preparation", "example_validation"] | Processing mode. Use `train_preparation` to prepare train and validation datasets for training, or `example_validation` to validate example data for PR submission. | -| `should_download` | bool | Whether to automatically download missing datasets from remote registries. Default: `False`. | -| `overwrite_metrics_conflicts` | bool | Whether or not to overwrite metrics conflicts. Default: `False`. | - -**Example** +| Option | Description | +| --- | --- | +| `--storage {hf,gitlab}` | Storage backend. Default: `hf`. | +| `--repo-id` | HF repo id, such as `org/dataset` (HF only). | +| `--name` | Dataset name (GitLab only). | +| `--revision` | Dataset version (GitLab only). | +| `--artifact` | Remote file to fetch (GitLab: required; HF: optional raw file). | +| `--output`, `-o` | Local destination file. | +| `--output-dir` | Local destination directory; needed when downloading all splits (HF only). | +| `--split` | Dataset split (HF only). | ```bash -config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/example_multi_step \ - +mode=example_validation +# Download a single file from HuggingFace +gym dataset download --repo-id NVIDIA/NeMo-Gym-Math-example_multi_step-v1 \ + --artifact train.jsonl \ + --output data/train.jsonl + +# Download from GitLab +gym dataset download \ + --storage gitlab \ + --name example_multi_step \ + --revision 0.0.1 \ + --artifact train.jsonl \ + --output data/train.jsonl ``` ---- - -## Dataset Registry - GitLab - -Commands for uploading, downloading, and managing datasets in GitLab Model Registry. +### `gym dataset rm` -### `ng_upload_dataset_to_gitlab` / `nemo_gym_upload_dataset_to_gitlab` +Delete a dataset from the GitLab Model Registry. Prompts for confirmation. Replaces `ng_delete_dataset_from_gitlab`. -Upload a local JSONL dataset artifact to GitLab. - -**Parameters** - -| Parameter | Type | Description | -| --- | --- | --- | -| `dataset_name` | str | The dataset name. | -| `version` | str | The version of this dataset. Must be in the format `x.x.x`. | -| `input_jsonl_fpath` | str | Path to the JSONL file to upload. | - -**Example** +| Option | Description | +| --- | --- | +| `--name` | Name of the dataset to delete. | ```bash -ng_upload_dataset_to_gitlab \ - +dataset_name=example_multi_step \ - +version=0.0.1 \ - +input_jsonl_fpath=data/train.jsonl +gym dataset rm --name old_dataset ``` ---- - -### `ng_download_dataset_from_gitlab` / `nemo_gym_download_dataset_from_gitlab` - -Download a JSONL dataset from GitLab Model Registry. +### `gym dataset migrate` -**Parameters** +Upload a local JSONL dataset to HuggingFace and delete it from GitLab afterward. Use `gym dataset upload` if you want optional rather than automatic GitLab deletion. Replaces `ng_gitlab_to_hf_dataset`. -| Parameter | Type | Description | -| --- | --- | --- | -| `dataset_name` | str | The dataset name. | -| `version` | str | The version of this dataset. Must be in the format `x.x.x`. | -| `artifact_fpath` | str | The filepath to the artifact to download. | -| `output_fpath` | str | Path where the downloaded dataset will be saved. | - -**Example** +| Option | Description | +| --- | --- | +| `--input`, `-i` | Local JSONL file to upload to HF. | +| `--name` | Dataset name. | +| `--revision` | Dataset revision (HF). | +| `--split` | Dataset split. | +| `--create-pr` | Open a pull request instead of committing directly. | ```bash -ng_download_dataset_from_gitlab \ - +dataset_name=example_multi_step \ - +version=0.0.1 \ - +artifact_fpath=train.jsonl \ - +output_fpath=data/train.jsonl +gym dataset migrate \ + --name my_dataset \ + --input data/train.jsonl \ + --revision 0.0.1 ``` ---- - -### `ng_delete_dataset_from_gitlab` / `nemo_gym_delete_dataset_from_gitlab` - -Delete a dataset from GitLab Model Registry. Prompts for confirmation. +### `gym dataset render` -**Parameters** +Generate a dataset preview by materializing prompts from a raw input file and a prompt template. Replaces `ng_materialize_prompts`. -| Parameter | Type | Description | -| --- | --- | --- | -| `dataset_name` | str | Name of the dataset to delete from GitLab. | - -**Example** +| Option | Description | +| --- | --- | +| `--input`, `-i` | Raw input JSONL file. | +| `--prompt-config` | Prompt template YAML to apply. | +| `--output`, `-o` | Output JSONL file. | ```bash -ng_delete_dataset_from_gitlab +dataset_name=old_dataset +gym dataset render \ + --input raw.jsonl \ + --prompt-config prompt.yaml \ + --output preview.jsonl ``` ---- - -## Dataset Registry - HuggingFace - -Commands for uploading and downloading datasets to/from HuggingFace Hub. - -### `ng_upload_dataset_to_hf` / `nemo_gym_upload_dataset_to_hf` - -Upload a JSONL dataset to HuggingFace Hub with optional GitLab deletion after successful upload. +### `gym dataset collate` -**Parameters** +Validate and collate a dataset, generating metrics and statistics. Replaces `ng_prepare_data`. -| Parameter | Type | Description | -| --- | --- | --- | -| `hf_token` | str | HuggingFace API token for authentication. | -| `hf_organization` | str | HuggingFace organization name where the dataset will be uploaded. | -| `hf_collection_name` | str | HuggingFace collection name for organizing datasets. | -| `hf_collection_slug` | str | Alphanumeric collection slug found at the end of the collection URI. | -| `dataset_name` | str | Name of the dataset. Will be combined with domain and resources server name. | -| `input_jsonl_fpath` | str | Path to the local JSONL file to upload. | -| `resource_config_path` | str | Path to resources server config file. Used to extract domain for naming convention. | -| `hf_dataset_prefix` | str | Prefix prepended to dataset name. Default: `NeMo-Gym`. | -| `delete_from_gitlab` | Optional[bool] | Delete the dataset from GitLab after successful upload to HuggingFace. Default: `False`. | - -**Example** +| Option | Description | +| --- | --- | +| `--config PATH` | Config file to load. Repeatable. | +| `--resource-server NAME` | Load the named resources server config. | +| `--search-dir DIR` | Extra root directory to search for named components. Repeatable. | +| `--mode {train_preparation,example_validation}` | Use `train_preparation` to prepare train/validation datasets, or `example_validation` to validate example data for PR submission. | +| `--output-dir` | Output directory for the prepared data. | +| `--download` | Download source datasets before collating. | ```bash -resource_config_path="resources_servers/example_multi_step/configs/example_multi_step.yaml" -ng_upload_dataset_to_hf \ - +dataset_name=my_dataset \ - +input_jsonl_fpath=data/train.jsonl \ - +resource_config_path=${resource_config_path} \ - +delete_from_gitlab=true +gym dataset collate \ + --resource-server example_multi_step \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/example_multi_step \ + --mode example_validation ``` --- -### `ng_download_dataset_from_hf` / `nemo_gym_download_dataset_from_hf` +## Environments -Download a JSONL dataset from HuggingFace Hub to local filesystem. +Commands for developing, running, and inspecting environments (a dataset + agent harness + resources server + model server). -**Parameters** +### `gym env init` -| Parameter | Type | Description | -| --- | --- | --- | -| `output_fpath` | str | Local file path where the downloaded dataset will be saved. | -| `hf_token` | str | HuggingFace API token for authentication. | -| `artifact_fpath` | str | Name of the artifact file to download from the repository. | -| `repo_id` | str | HuggingFace repository ID in format `organization/dataset-name`. | +Scaffold a new resources server with template files (config, app, tests, README, data directory). Replaces `ng_init_resources_server`. -**Example** +| Option | Description | +| --- | --- | +| `--resource-server NAME` | Name of the resources server to create. | ```bash -ng_download_dataset_from_hf \ - +repo_id=NVIDIA/NeMo-Gym-Math-example_multi_step-v1 \ - +artifact_fpath=train.jsonl \ - +output_fpath=data/train.jsonl +gym env init --resource-server my_server ``` ---- +### `gym env resolve` -### `ng_gitlab_to_hf_dataset` / `nemo_gym_gitlab_to_hf_dataset` +Resolve the configs, flags, and overrides into a final merged config and print it. Useful for debugging configuration. Secrets are hidden. Replaces `ng_dump_config`. -Upload a JSONL dataset to HuggingFace Hub and automatically delete from GitLab after successful upload. +| Option | Description | +| --- | --- | +| `--config PATH` | Config file to load. Repeatable. | -This command always deletes the dataset from GitLab after uploading to HuggingFace. Use `ng_upload_dataset_to_hf` if you want optional deletion control. +```bash +gym env resolve --config resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml +``` -**Parameters** +### `gym env packages` -Same as `ng_upload_dataset_to_hf` but `delete_from_gitlab` is not available. This command always deletes. +Each server has its own isolated virtual environment. List the packages installed in a server's environment. Replaces `ng_pip_list`. -**Example** +| Option | Description | +| --- | --- | +| `--resource-server NAME` | Name of the resources server. | +| `--outdated` | List only outdated packages. | +| `--json` | Output the package list as JSON. | ```bash -resource_config_path="resources_servers/example_multi_step/configs/example_multi_step.yaml" -ng_gitlab_to_hf_dataset \ - +dataset_name=my_dataset \ - +input_jsonl_fpath=data/train.jsonl \ - +resource_config_path=${resource_config_path} -``` - ---- - -## Configuration & Help +gym env packages --resource-server example_single_tool_call -Commands for debugging configuration and getting help. +# Check for outdated packages +gym env packages \ + --resource-server example_single_tool_call \ + --outdated +``` -### `ng_dump_config` / `nemo_gym_dump_config` +### `gym env test` -Display the resolved Hydra configuration for debugging purposes. +Test resource server(s) by running their pytest suite. If no resources server is given, all of them are tested. Replaces `ng_test` and `ng_test_all`. -**Example** +| Option | Description | +| --- | --- | +| `--resource-server NAME` | Resources server to test. Omit to test all servers. | ```bash -ng_dump_config "+config_paths=[,]" -``` +# Test a single server +gym env test --resource-server example_single_tool_call ---- +# Test all servers +gym env test +``` -### `ng_help` / `nemo_gym_help` +### `gym env run` -Display a list of available NeMo Gym CLI commands. +Start the NeMo Gym servers (agents, models, resources) defined by the provided configs. Reads configuration from YAML files and runs each configured server in its own environment. Replaces `ng_run`. -**Example** +| Option | Description | +| --- | --- | +| `--config PATH` | Config file to load. Repeatable. | +| `--benchmark NAME` | Load the named benchmark config (start its servers). | +| `--resource-server NAME` | Load the named resources server config. | +| `--model-type NAME` | Load the named model server type config. | +| `--search-dir DIR` | Extra root directory to search for named components. Repeatable. | +| `--model`, `-m` | Served model identifier. See [Selecting the model server](#selecting-the-model-server). | +| `--model-url` | Model server base URL. | +| `--model-api-key` | Model server API key. | ```bash -ng_help +gym env run \ + --resource-server example_single_tool_call \ + --model-type openai_model + +# Start a benchmark's servers +gym env run \ + --benchmark gpqa \ + --model-type vllm_model ``` ---- - -### `ng_version` / `nemo_gym_version` +### `gym env status` -Display NeMo Gym version and system information. +Show all currently running NeMo Gym servers and their health. Replaces `ng_status`. -**Parameters** +| Option | Description | +| --- | --- | +| `--json` | Output the server list as JSON. | -| Parameter | Type | Description | -| --- | --- | --- | -| `json_format` | bool | Output in JSON format for programmatic use. Default: `False`. Can be specified with `+json=true`. | +```bash +gym env status +``` -**Example** +``` +NeMo Gym Server Status: -```bash -# Display version information -ng_version +[1] ✓ example_single_tool_call (resources_servers/example_single_tool_call) +{ + 'server_type': 'resources_servers', + 'name': 'example_single_tool_call', + 'port': 58117, + 'pid': 89904, + 'uptime_seconds': '0d 0h 0m 41.5s', +} +... -# Output as JSON -ng_version +json=true +3 servers found (3 healthy, 0 unhealthy) ``` --- -### `ng_pip_list` / `nemo_gym_pip_list` +## Evaluation -Each server has its own isolated virtual environment. To inspect the packages: +Commands for running evaluations end to end: prepare data, collect rollouts, aggregate, and profile. -**Parameters** +### `gym eval prepare` -| Parameter | Type | Description | -| --- | --- | --- | -| `entrypoint` | str | The relative entrypoint path to the server directory | -| `format` | Optional[str] | Output format for pip list. Options: 'columns' (default), 'freeze', 'json'. Default: `None`. | -| `outdated` | bool | List outdated packages. Default: `False`. | +Prepare a benchmark's data by running its `prepare.py` script and dump the result to disk. Replaces `ng_prepare_benchmark`. -**Examples** +| Option | Description | +| --- | --- | +| `--config PATH` | Config file to load. Repeatable. | +| `--benchmark NAME` | Load the named benchmark config. | +| `--search-dir DIR` | Extra root directory to search for named components. Repeatable. | ```bash -# List all packages -ng_pip_list +entrypoint=resources_servers/example_single_tool_call +gym eval prepare --benchmark aime24 +``` -# Output as JSON -ng_pip_list +entrypoint=resources_servers/example_single_tool_call +format=json +### `gym eval run` + +Collate data, start the servers, and collect rollouts. This is the main evaluation command. By default it spins up all required servers (the former `ng_e2e_collect_rollouts`). Pass `--no-serve` to collect against servers you already started with `gym env run` (the former `ng_collect_rollouts`). Replaces `ng_e2e_collect_rollouts` and `ng_collect_rollouts`. + +| Option | Description | +| --- | --- | +| `--config PATH` | Config file to load. Repeatable. | +| `--benchmark NAME` | Load the named benchmark config. | +| `--resource-server NAME` | Load the named resources server config. | +| `--model-type NAME` | Load the named model server type config. | +| `--search-dir DIR` | Extra root directory to search for named components. Repeatable. | +| `--no-serve` | Collect against already-running servers instead of starting them. | +| `--resume` | Resume from cached rollouts instead of recollecting. | +| `--agent`, `-a` | Agent to collect rollouts with. | +| `--input`, `-i` | Input tasks JSONL file. | +| `--output`, `-o` | Output rollouts JSONL file. | +| `--limit` | Maximum number of tasks to run. | +| `--num-repeats` | Number of rollouts per task (for mean@k metrics). | +| `--prompt-config` | Prompt template YAML to apply. | +| `--concurrency` | Maximum number of concurrent samples. | +| `--split` | Dataset split to use (`train`, `validation`, or `benchmark`). | +| `--model`, `-m` | Served model identifier. | +| `--model-url` | Model server base URL. | +| `--model-api-key` | Model server API key. | +| `--temperature` | Sampling temperature. | +| `--top-p` | Nucleus sampling top-p. | +| `--max-output-tokens` | Maximum output tokens. | -# Check for outdated packages -ng_pip_list +entrypoint=resources_servers/example_single_tool_call +outdated=true +```bash +# End-to-end: spin up servers, then collect rollouts for a benchmark +gym eval run --benchmark aime24 \ + --model-type openai_model \ + --output results/aime24.jsonl \ + --split validation \ + --concurrency 10 + +# Against an already-running server, with a remote vLLM endpoint +gym eval run --no-serve \ + --model-type openai_model \ + --resource-server math_with_judge \ + --output results/test_001.jsonl \ + --split validation \ + --model openai/gpt-oss-120b \ + --model-url http://0.0.0.0:10240/v1 \ + --model-api-key dummy_key \ + --temperature 1.0 \ + --top-p 1.0 ``` ---- - -### `ng_status` / `nemo_gym_status` +### `gym eval aggregate` -View all currently running NeMo Gym servers and their health status. +Merge sharded rollout results into a single rollouts file with aggregate metrics. Replaces `ng_aggregate_rollouts`. -**Example** +| Option | Description | +| --- | --- | +| `--config PATH` | Config file to load. Repeatable. | +| `--output`, `-o` | Path for the merged rollouts and aggregate-metrics file. | ```bash -ng_status +gym eval aggregate --output results/merged.jsonl +``` -NeMo Gym Server Status: +### `gym eval profile` -[1] ✓ example_single_tool_call (resources_servers/example_single_tool_call) -{ - 'server_type': 'resources_servers', - 'name': 'example_single_tool_call', - 'port': 58117, - 'pid': 89904, - 'uptime_seconds': '0d 0h 0m 41.5s', -} -[2] ✓ example_single_tool_call_simple_agent (responses_api_agents/simple_agent) -{ - 'server_type': 'responses_api_agents', - 'name': 'simple_agent', - 'port': 58118, - 'pid': 89905, - 'uptime_seconds': '0d 0h 0m 41.5s', -} -[3] ✓ policy_model (responses_api_models/openai_model) -{ - 'server_type': 'responses_api_models', - 'name': 'openai_model', - 'port': 58119, - 'pid': 89907, - 'uptime_seconds': '0d 0h 0m 41.5s', -} +Compute a reward profile from collected rollouts. Outputs per-task statistics such as average reward, standard deviation, min/max, and pass rate, useful for filtering tasks before training by difficulty or variance. Requires rollouts collected with `--num-repeats` greater than 1. Replaces `ng_reward_profile`. -3 servers found (3 healthy, 0 unhealthy) +| Option | Description | +| --- | --- | +| `--inputs` | Materialized inputs JSONL fed to rollout collection. | +| `--rollouts` | Rollouts JSONL produced by collection. | +```bash +gym eval profile \ + --inputs materialized_inputs.jsonl \ + --rollouts rollouts.jsonl ``` --- -## Getting Help +## Contributor Helpers -For detailed help on any command, run it with `+help=true` or `+h=true`: +### `gym dev test` + +Run NeMo Gym's core unit tests with coverage reporting. Replaces `ng_dev_test`. ```bash -ng_run +help=true -ng_collect_rollouts +h=true +gym dev test ``` -This will display all available configuration parameters and their descriptions. - --- -## Re-install Gym and dependencies +## Migrating from the legacy commands + +The legacy `ng_*` and `nemo_gym_*` console scripts are deprecated but still functional. Each prints a one-time deprecation notice that names its `gym` replacement, then forwards your arguments (including any Hydra overrides) to the new command. Scripts, CI pipelines, and muscle memory keep working during the deprecation period; update them to the `gym` syntax when convenient. + +### Command mapping + +| Legacy command | New command | +| --- | --- | +| `ng_help` | `gym --help` | +| `ng_version` | `gym --version` | +| `ng_list_benchmarks` | `gym list benchmarks` | +| `ng_run` | `gym env run` | +| `ng_status` | `gym env status` | +| `ng_dump_config` | `gym env resolve` | +| `ng_pip_list` | `gym env packages` | +| `ng_init_resources_server` | `gym env init` | +| `ng_test` | `gym env test` | +| `ng_test_all` | `gym env test` (no `--resource-server`) | +| `ng_prepare_benchmark` | `gym eval prepare` | +| `ng_e2e_collect_rollouts` | `gym eval run` | +| `ng_collect_rollouts` | `gym eval run --no-serve` | +| `ng_aggregate_rollouts` | `gym eval aggregate` | +| `ng_reward_profile` | `gym eval profile` | +| `ng_prepare_data` | `gym dataset collate` | +| `ng_materialize_prompts` | `gym dataset render` | +| `ng_upload_dataset_to_hf` | `gym dataset upload` | +| `ng_upload_dataset_to_gitlab` | `gym dataset upload --storage gitlab` | +| `ng_download_dataset_from_hf` | `gym dataset download` | +| `ng_download_dataset_from_gitlab` | `gym dataset download --storage gitlab` | +| `ng_gitlab_to_hf_dataset` | `gym dataset migrate` | +| `ng_delete_dataset_from_gitlab` | `gym dataset rm` | +| `ng_dev_test` | `gym dev test` | +| `ng_reinstall` | `uv sync --extra dev --group docs` | + +### Replacing Hydra overrides with flags + +The most common Hydra overrides now have dedicated flags. Prefer flags; fall back to `+key=value` / `++key=value` for anything without one. + +| Legacy Hydra override | New flag | +| --- | --- | +| `"+config_paths=[a.yaml,b.yaml]"` | `--config a.yaml --config b.yaml` | +| `+agent_name=...` | `--agent`, `-a` | +| `+input_jsonl_fpath=...` | `--input`, `-i` | +| `+output_jsonl_fpath=...` | `--output`, `-o` | +| `+limit=...` | `--limit` | +| `+num_repeats=...` | `--num-repeats` | +| `+num_samples_in_parallel=...` | `--concurrency` | +| `++split=...` | `--split` | +| `++policy_model_name=...` | `--model`, `-m` | +| `++policy_base_url=...` | `--model-url` | +| `++policy_api_key=...` | `--model-api-key` | +| `++responses_create_params.temperature=...` | `--temperature` | +| `++responses_create_params.top_p=...` | `--top-p` | +| `++responses_create_params.max_output_tokens=...` | `--max-output-tokens` | +| `+mode=...` | `--mode` | +| `+output_dirpath=...` | `--output-dir` | +| `+should_download=true` | `--download` | + +### Common workflows, before and after + ```bash -ng_reinstall +# Start servers +# Before: +ng_run "+config_paths=[resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +# After: +gym env run \ + --resource-server example_single_tool_call \ + --model-type openai_model + +# End-to-end rollout collection +# Before: +config_paths="responses_api_models/openai_model/configs/openai_model.yaml,resources_servers/math_with_judge/configs/math_with_judge.yaml" +ng_e2e_collect_rollouts "+config_paths=[${config_paths}]" \ + ++output_jsonl_fpath=results/aime24.jsonl \ + ++split=validation +# After: +gym eval run \ + --model-type openai_model \ + --resource-server math_with_judge \ + --output results/aime24.jsonl \ + --split validation + +# Collect against already-running servers +# Before: +ng_collect_rollouts +agent_name=example_single_tool_call_simple_agent \ + +input_jsonl_fpath=weather_query.jsonl \ + +output_jsonl_fpath=weather_rollouts.jsonl \ + +num_repeats=4 +num_samples_in_parallel=10 +# After: +gym eval run --no-serve \ + --agent example_single_tool_call_simple_agent \ + --input weather_query.jsonl \ + --output weather_rollouts.jsonl \ + --num-repeats 4 \ + --concurrency 10 + +# Reward profiling +# Before: +ng_reward_profile +input_jsonl_fpath=materialized_inputs.jsonl +rollouts_jsonl_fpath=rollouts.jsonl +# After: +gym eval profile \ + --inputs materialized_inputs.jsonl \ + --rollouts rollouts.jsonl ``` -This will re-install Gym and its dependencies into the currently activated Python virtual environment. + +Run any command with `--help` to see its full set of flags, and `gym --help` to list every group. + diff --git a/fern/versions/latest/pages/reference/configuration.mdx b/fern/versions/latest/pages/reference/configuration.mdx index 300d51b8fc..1cdda4ae3e 100644 --- a/fern/versions/latest/pages/reference/configuration.mdx +++ b/fern/versions/latest/pages/reference/configuration.mdx @@ -135,31 +135,36 @@ error_on_almost_servers: true # Exit on invalid configs (default: true) ## Command Line Usage -To run servers, use `ng_run`. NeMo Gym uses [Hydra](https://hydra.cc/) for configuration management. +To run servers, use `gym env run`. NeMo Gym uses [Hydra](https://hydra.cc/) for configuration management. ### Loading Configs ```bash # Load one or more config files -ng_run "+config_paths=[config1.yaml,config2.yaml]" +gym env run \ + --config config1.yaml \ + --config config2.yaml # Use paths stored in env.yaml -ng_run "+config_paths=${my_config_paths}" +gym env run "+config_paths=${my_config_paths}" ``` ### Overriding Values ```bash # Override nested values (use dot notation after server ID) -ng_run "+config_paths=[config.yaml]" \ +gym env run \ + --config config.yaml \ +my_server.resources_servers.my_impl.domain=coding # Override policy model -ng_run "+config_paths=[config.yaml]" \ - +policy_model_name=gpt-4o-mini +gym env run --config config.yaml \ + --model gpt-4o-mini # Disable strict validation -ng_run "+config_paths=[config.yaml]" +error_on_almost_servers=false +gym env run \ + --config config.yaml \ + +error_on_almost_servers=false ``` --- diff --git a/fern/versions/latest/pages/reference/faq.mdx b/fern/versions/latest/pages/reference/faq.mdx index de86b3ea2e..b4e8f6538e 100644 --- a/fern/versions/latest/pages/reference/faq.mdx +++ b/fern/versions/latest/pages/reference/faq.mdx @@ -39,9 +39,9 @@ You will only need to manually input the `{dataset_name}` portion of the above w To upload to Hugging Face, use the below command: ```bash resource_config_path="resources_servers/multineedle/configs/multineedle.yaml" -ng_upload_dataset_to_hf \ - +dataset_name={your dataset name} \ - +input_jsonl_fpath=data/multineedle_benchmark.jsonl \ +gym dataset upload \ + --name {your dataset name} \ + --input data/multineedle_benchmark.jsonl \ +resource_config_path=${resource_config_path} ``` @@ -51,11 +51,11 @@ By default, the `split` parameter for uploading is set to `train`, which will ru ```bash resource_config_path="resources_servers/multineedle/configs/multineedle.yaml" -ng_gitlab_to_hf_dataset \ - +dataset_name={your dataset name} \ - +input_jsonl_fpath=data/multineedle_benchmark_validation.jsonl \ - +resource_config_path=${resource_config_path} \ - +split=validation +gym dataset migrate \ + --name {your dataset name} \ + --input data/multineedle_benchmark_validation.jsonl \ + --split validation \ + +resource_config_path=${resource_config_path} ``` ## Uploading with Pull Request workflow @@ -64,13 +64,13 @@ When uploading to an organization repository where you don't have direct write a If you want to specify the revision (branch name), you can add the `+revision={your branch name}` flag. Excluding `create_pr` (or setting it to `false`) assumes you are committing to an existing branch. Including it assumes it will be a brand new branch. ```bash -ng_upload_dataset_to_hf \ - +dataset_name=OpenMathReasoning \ - +input_jsonl_fpath=data/validation.jsonl \ +gym dataset upload \ + --name OpenMathReasoning \ + --input data/validation.jsonl \ + --split validation \ + --revision my-branch-name \ + --create-pr \ +resource_config_path=${resource_config_path} \ - +split=validation \ - +create_pr=true \ - +revision=my-branch-name \ +commit_message="Add validation set" \ +commit_description="Includes 545 examples" ``` @@ -89,7 +89,7 @@ The commit_message and commit_description parameters work for both direct pushes You can optionally pass a `+delete_from_gitlab=true` flag to the above command, which will delete the model and all of its artifacts from GitLab. By default, this is set to `False`. ```bash resource_config_path="resources_servers/multineedle/configs/multineedle.yaml" -ng_upload_dataset_to_hf \ +gym dataset upload \ +dataset_name={your dataset name} \ +input_jsonl_fpath=data/multineedle_benchmark.jsonl \ +resource_config_path=${resource_config_path} \ @@ -106,16 +106,16 @@ You can also run the below command which does the same thing without the need fo ```bash resource_config_path="resources_servers/multineedle/configs/multineedle.yaml" -ng_gitlab_to_hf_dataset \ - +dataset_name={your dataset name} \ - +input_jsonl_fpath=data/multineedle_benchmark.jsonl \ +gym dataset migrate \ + --name {your dataset name} \ + --input data/multineedle_benchmark.jsonl \ +resource_config_path=${resource_config_path} ``` If you've already uploaded to Hugging Face and just want to do a standalone delete from GitLab: ```bash -ng_delete_dataset_from_gitlab \ - +dataset_name={your dataset name} +gym dataset rm \ + --name {your dataset name} ``` @@ -128,24 +128,24 @@ Downloading a dataset from Hugging Face is straightforward: **For structured datasets (with train/validation/test splits):** ```bash -ng_download_dataset_from_hf \ - +repo_id=nvidia/Nemotron-RL-knowledge-mcqa \ - +output_dirpath=data/mcqa \ - +split=train +gym dataset download \ + --repo-id nvidia/Nemotron-RL-knowledge-mcqa \ + --output-dir data/mcqa \ + --split train ``` The `split` parameter is optional. If omitted, all available splits will be downloaded as separate JSONL files. **For raw file repositories (with specific JSONL files):** ```bash -ng_download_dataset_from_hf \ - +repo_id=nvidia/Nemotron-RL-instruction_following \ - +output_dirpath=data/instruction_following \ - +artifact_fpath=instruction_following.jsonl +gym dataset download \ + --repo-id nvidia/Nemotron-RL-instruction_following \ + --output-dir data/instruction_following \ + --artifact instruction_following.jsonl ``` Use `artifact_fpath` when the Hugging Face repo contains raw/arbitrary JSONL files rather than structured dataset splits. You cannot specify both `split` and `artifact_fpath`. # How To: Prepare and validate data for PR submission or RL training -When you use `ng_init_resources_server +entrypoint=resources_servers/example_multi_step` to initialize a resources server, you will get a config.yaml that looks like the below code block. The dataset information for training, validation, and example will be inside the scope of your agent config (e.g. under simple_agent) and is a list of dataset objects. +When you use `gym env init --resource-server example_multi_step` to initialize a resources server, you will get a config.yaml that looks like the below code block. The dataset information for training, validation, and example will be inside the scope of your agent config (e.g. under simple_agent) and is a list of dataset objects. ```yaml example_multi_step_resources_server: @@ -220,39 +220,45 @@ A dataset object consists of: Each config.yaml in the resources server requires at least one agent with one example dataset. This example dataset is the first 5 rows of your train dataset that is used for sanity checks on the format for your dataset and the format of each individual example and for others to quickly understand your data. -For every PR that contributes data, we require common dataset statistics and sanity checks on the data itself. This process is also helpful to catch any simple issues before you ever train with NeMo RL. NeMo Gym provides a helper command ng_prepare_data to do so. +For every PR that contributes data, we require common dataset statistics and sanity checks on the data itself. This process is also helpful to catch any simple issues before you ever train with NeMo RL. NeMo Gym provides a helper command gym dataset collate to do so. ```bash -config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_prepare_data "+config_paths=[$config_paths]" \ - +output_dirpath=data/example_multi_step \ - +mode=example_validation +gym dataset collate \ + --resource-server example_multi_step \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/example_multi_step \ + --mode example_validation ``` -To download missing datasets automatically, add +should_download=true. By default, datasets are downloaded from Hugging Face: +To download missing datasets automatically, add --download. By default, datasets are downloaded from Hugging Face: ```bash -ng_prepare_data "+config_paths=[$config_paths]" \ - +output_dirpath=data/example_multi_step \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --resource-server example_multi_step \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/example_multi_step \ + --mode train_preparation \ + --download ``` For NVIDIA internal users, you can download from GitLab instead: ```bash -ng_prepare_data "+config_paths=[$config_paths]" \ - +output_dirpath=data/example_multi_step \ - +mode=train_preparation \ - +should_download=true \ +gym dataset collate \ + --resource-server example_multi_step \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/example_multi_step \ + --mode train_preparation \ + --download \ +data_source=gitlab ``` Run NeMo Gym servers the exact same way with the same configs! ```bash -ng_run "+config_paths=[$config_paths]" +gym env run \ + --resource-server example_multi_step \ + --model-type openai_model ``` -The `ng_prepare_data` command will: +The `gym dataset collate` command will: 1. Attempt to load all the datasets you specified from disk. Missing datasets will be reported before any processing is done. 2. For each dataset, read example by example. Check the format and report the filepaths and indices/ranges of offending examples if any. 1. We only require that the dataset has one key responses_create_params which is valid Responses API schema. @@ -267,15 +273,15 @@ The `ng_prepare_data` command will: 4. Check that the aggregate statistics of individual datasets match those of existing aggregate statistics. 5. Collate all the examples into one final train and validation dataset jsonl files at the output dirpath specified for downstream NeMo RL or other train framework consumption. 6. The final aggregate statistics are reported and saved next to the train and validation datasets. -7. [NeMo RL train] Use the exact same config paths to ng_prepare_data and the train/validation dataset paths output in step 5. There is no special pre or post processing done in the NeMo Gym/RL integration other than shuffling and distributed data loading. What you see is what you get. +7. [NeMo RL train] Use the exact same config paths to gym dataset collate and the train/validation dataset paths output in step 5. There is no special pre or post processing done in the NeMo Gym/RL integration other than shuffling and distributed data loading. What you see is what you get. -The `ng_prepare_data` command has 2 modes, one for actual train and validation set preparation, and one for example validation intended to sanity check your data format. You would typically run `+mode=example_validation` when first contributing a resources server, and then run with `+mode=train_preparation` when you actually go to train. +The `gym dataset collate` command has 2 modes, one for actual train and validation set preparation, and one for example validation intended to sanity check your data format. You would typically run `--mode example_validation` when first contributing a resources server, and then run with `--mode train_preparation` when you actually go to train. ```bash -config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_prepare_data "+config_paths=[$config_paths]" \ - +output_dirpath=data/example_multi_step \ - +mode=example_validation +gym dataset collate \ + --resource-server example_multi_step \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/example_multi_step \ + --mode example_validation ``` # How To: Profile your resources server @@ -285,23 +291,24 @@ In one terminal, start your agent, model, and resources servers, with profiling - `profiling_enabled` (bool): whether profiling is enabled or not. By default this is disabled since it incurs some slight overhead we don't want at runtime. - `profiling_results_dirpath` (str): The directory to save all server profiling results in. Previous logs for the same will be overwritten in the same directory. ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/math_with_judge/configs/bytedtsinghua_dapo17k.yaml" -ng_run "+config_paths=[${config_paths}]" \ +gym env run \ + --model-type openai_model \ + --config resources_servers/math_with_judge/configs/bytedtsinghua_dapo17k.yaml \ +profiling_enabled=true \ +profiling_results_dirpath=results/profiling/math_with_judge ``` In another terminal, run some large number of rollouts against your servers. Use the `limit` and `num_repeats` flags to adjust the number of samples you want to run. ```bash -ng_collect_rollouts +agent_name=math_with_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/math_with_judge/data/dapo17k_bytedtsinghua_train.jsonl \ - +output_jsonl_fpath=temp/math_with_judge_rollouts.jsonl \ - +limit=1024 \ - +num_repeats=1 +gym eval run --no-serve \ + --agent math_with_judge_simple_agent \ + --input resources_servers/math_with_judge/data/dapo17k_bytedtsinghua_train.jsonl \ + --output temp/math_with_judge_rollouts.jsonl \ + --limit 1024 \ + --num-repeats 1 ``` -After `ng_collect_rollouts` finishes, ctrl+c to quit your servers. You should see some output in the terminal like the following: +After `gym eval run` finishes, ctrl+c to quit your servers. You should see some output in the terminal like the following: ```bash ``` @@ -334,7 +341,7 @@ NeMo Gym automatically sets up Ray for distributed computing for CPU-intensive t Ray is initialized when you start NeMo Gym servers: ```bash -ng_run "+config_paths=[$config_paths]" +gym env run --config $config_paths ``` The initialization happens in two places: @@ -393,7 +400,7 @@ Tying back to NeMo Gym, NeMo Gym can be used to create synthetic data for SFT tr # FAQ: PermissionError when starting NeMo Gym in sandboxed environments -If you see an error like the following when running `ng_run`: +If you see an error like the following when running `gym env run`: ```python PermissionError: [Errno 1] Operation not permitted (originated from sysctl() malloc 1/3) @@ -424,7 +431,7 @@ If you're running NeMo Gym in a sandboxed environment and hit this error, you'll 2. **Grant additional permissions** to allow Ray to access process information 3. **Run outside the sandbox** in a normal terminal environment -For most development and production use cases, simply running `ng_run` in your regular terminal will work without any issues. +For most development and production use cases, simply running `gym env run` in your regular terminal will work without any issues. **Specific workaround for Cursor AI:** diff --git a/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx b/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx index 535b42592c..5a114aaa50 100644 --- a/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx +++ b/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx @@ -13,24 +13,24 @@ Suppose you want to use both the example_single_tool_call and example_multi_step For example_single_tool_call: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --model-type openai_model \ + --resource-server example_single_tool_call ``` For example_multi_step: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/example_multi_step/configs/example_multi_step.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --resource-server example_multi_step ``` To use both environments, add the YAML configs together as follows: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,\ -resources_servers/example_multi_step/configs/example_multi_step.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --resource-server example_single_tool_call \ + --resource-server example_multi_step ``` ## Dataset Preparation @@ -45,9 +45,9 @@ jq -c '. + {"agent_ref": {"type": "responses_api_agents", "name": "example_multi Run rollout collection as usual. ```bash -ng_collect_rollouts \ - +input_jsonl_fpath=results/test_multiverifier_input.jsonl \ - +output_jsonl_fpath=results/test_multiverifier_outputs.jsonl +gym eval run --no-serve \ + --input results/test_multiverifier_input.jsonl \ + --output results/test_multiverifier_outputs.jsonl ``` Inside `results/test_multiverifier_outputs.jsonl`, you should see 10 rows with appropriate responses for each row. diff --git a/fern/versions/latest/pages/training-tutorials/nemo-rl-grpo/setup.mdx b/fern/versions/latest/pages/training-tutorials/nemo-rl-grpo/setup.mdx index c3be4ac326..e909e78650 100644 --- a/fern/versions/latest/pages/training-tutorials/nemo-rl-grpo/setup.mdx +++ b/fern/versions/latest/pages/training-tutorials/nemo-rl-grpo/setup.mdx @@ -149,7 +149,7 @@ uv run python -c "import ray; ray.shutdown()" **Estimated time**: ~5 minutes -The Workplace Assistant dataset must be downloaded from HuggingFace and prepared for training. This runs `ng_prepare_data` to download and validate the dataset, and to add an `agent_ref` property to each example that tells NeMo Gym which agent server should handle that example. +The Workplace Assistant dataset must be downloaded from HuggingFace and prepared for training. This runs `gym dataset collate` to download and validate the dataset, and to add an `agent_ref` property to each example that tells NeMo Gym which agent server should handle that example. Clone and setup the Gym Python environment: @@ -170,13 +170,12 @@ echo "hf_token: {your HF token}" >> env.yaml Prepare the data: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/workplace_assistant/configs/workplace_assistant.yaml" - -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/workplace_assistant \ - +mode=train_preparation \ - +should_download=true \ +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resource-server workplace_assistant \ + --output-dir data/workplace_assistant \ + --mode train_preparation \ + --download \ +data_source=huggingface ``` diff --git a/fern/versions/latest/pages/training-tutorials/offline-training-w-rollouts.mdx b/fern/versions/latest/pages/training-tutorials/offline-training-w-rollouts.mdx index 46218bca15..dbaddd3ddf 100644 --- a/fern/versions/latest/pages/training-tutorials/offline-training-w-rollouts.mdx +++ b/fern/versions/latest/pages/training-tutorials/offline-training-w-rollouts.mdx @@ -267,9 +267,10 @@ Test your improved model by generating new rollouts on held-out evaluation tasks ```bash # Generate rollouts with improved model -ng_collect_rollouts +agent_name=improved_agent \ - +input_jsonl_fpath=evaluation_tasks.jsonl \ - +output_jsonl_fpath=post_training_rollouts.jsonl +gym eval run --no-serve \ + --agent improved_agent \ + --input evaluation_tasks.jsonl \ + --output post_training_rollouts.jsonl ``` Compare key metrics like average reward, success rate, and task-specific performance against your baseline to measure improvement. diff --git a/fern/versions/latest/pages/training-tutorials/verl.mdx b/fern/versions/latest/pages/training-tutorials/verl.mdx index b44cada8ec..ccef3cdfa3 100644 --- a/fern/versions/latest/pages/training-tutorials/verl.mdx +++ b/fern/versions/latest/pages/training-tutorials/verl.mdx @@ -36,14 +36,12 @@ Using NeMo Gym, prepare the training dataset for your environment. Each row need cd $NEMO_GYM_ROOT source .venv/bin/activate -config_paths="resources_servers/workplace_assistant/configs/workplace_assistant.yaml,\ -responses_api_models/vllm_model/configs/vllm_model_for_training.yaml" - -ng_prepare_data \ - "+config_paths=[${config_paths}]" \ - +output_dirpath=data/workplace_assistant \ - +mode=train_preparation \ - +should_download=true \ +gym dataset collate \ + --resource-server workplace_assistant \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --output-dir data/workplace_assistant \ + --mode train_preparation \ + --download \ +data_source=huggingface ``` diff --git a/fern/versions/latest/pages/troubleshooting/configuration.mdx b/fern/versions/latest/pages/troubleshooting/configuration.mdx index da1bb1e28d..d52a530db7 100644 --- a/fern/versions/latest/pages/troubleshooting/configuration.mdx +++ b/fern/versions/latest/pages/troubleshooting/configuration.mdx @@ -3,7 +3,7 @@ title: "Configuration Errors" description: "" position: 1 --- -These errors appear when running `ng_run` or `ng_collect_rollouts`. NeMo Gym validates configuration at startup before launching servers. +These errors appear when running `gym env run` or `gym eval run --no-serve`. NeMo Gym validates configuration at startup before launching servers. See the [Configuration reference](/reference/configuration) for complete configuration syntax and options. @@ -33,7 +33,9 @@ policy_api_key: sk-your-api-key Or pass via command line: ```bash -ng_run "+config_paths=[config.yaml]" +policy_api_key=sk-your-api-key +gym env run \ + --config config.yaml \ + --model-api-key sk-your-api-key ``` ### Server Reference Not Found @@ -47,15 +49,18 @@ in the list of available servers: [ResourcesServerRef(...), ...] **Common causes**: - Typo in the server name -- Referenced server's config file not included in `+config_paths` +- Referenced server's config file not included in the `--config` flags - Server defined in a different config file that wasn't loaded **Fix**: 1. Check server name spelling in your config -2. Ensure all required config files are in `+config_paths`: +2. Ensure all required config files are passed with `--config`: ```bash -ng_run "+config_paths=[model.yaml,resource.yaml,agent.yaml]" +gym env run \ + --config model.yaml \ + --config resource.yaml \ + --config agent.yaml ``` --- @@ -97,7 +102,9 @@ error_on_almost_servers: false ```bash # Or via command line -ng_run "+config_paths=[config.yaml]" +error_on_almost_servers=false +gym env run \ + --config config.yaml \ + +error_on_almost_servers=false ``` diff --git a/resources_servers/abstention/README.md b/resources_servers/abstention/README.md index c790f869fb..a51aa62f4a 100644 --- a/resources_servers/abstention/README.md +++ b/resources_servers/abstention/README.md @@ -15,20 +15,20 @@ The dataset used is [HotPotQA](https://hotpotqa.github.io/) (fullwiki split). ## Running servers ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -resources_servers/abstention/configs/abstention.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --model-type openai_model \ + --resource-server abstention \ +abstention.resources_servers.abstention.judge_model_server.name=policy_model ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=abstention_simple_agent \ - +input_jsonl_fpath=resources_servers/abstention/data/example.jsonl \ - +output_jsonl_fpath=results/abstention_verify_responses.jsonl \ - +limit=3 +gym eval run --no-serve \ + --agent abstention_simple_agent \ + --input resources_servers/abstention/data/example.jsonl \ + --output results/abstention_verify_responses.jsonl \ + --limit 3 ``` ## Preprocessing HotPotQA data diff --git a/resources_servers/arc_agi/README.md b/resources_servers/arc_agi/README.md index 08987a922b..aea6824f12 100644 --- a/resources_servers/arc_agi/README.md +++ b/resources_servers/arc_agi/README.md @@ -51,7 +51,9 @@ uv sync ### Start ARC-AGI environment (we can reuse the same one for ARC-AGI-1 and 2): ```bash -ng_run "+config_paths=[resources_servers/arc_agi/configs/arc_agi.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --resource-server arc_agi \ + --model-type vllm_model ``` @@ -59,12 +61,18 @@ ng_run "+config_paths=[resources_servers/arc_agi/configs/arc_agi.yaml,responses_ ARC-AGI-1 example rollouts: ```bash -ng_collect_rollouts +agent_name=arc_agi_simple_agent +input_jsonl_fpath=resources_servers/arc_agi/data/example_1.jsonl +output_jsonl_fpath=resources_servers/arc_agi/data/example_1_rollouts.jsonl +gym eval run --no-serve \ + --agent arc_agi_simple_agent \ + --input resources_servers/arc_agi/data/example_1.jsonl \ + --output resources_servers/arc_agi/data/example_1_rollouts.jsonl ``` ARC-AGI-2 example rollouts: ```bash -ng_collect_rollouts +agent_name=arc_agi_simple_agent +input_jsonl_fpath=resources_servers/arc_agi/data/example_2.jsonl +output_jsonl_fpath=resources_servers/arc_agi/data/example_2_rollouts.jsonl +gym eval run --no-serve \ + --agent arc_agi_simple_agent \ + --input resources_servers/arc_agi/data/example_2.jsonl \ + --output resources_servers/arc_agi/data/example_2_rollouts.jsonl ``` For training, see the [docs](https://docs.nvidia.com/nemo/gym/latest/training-tutorials/nemo-rl-grpo/index.html). \ No newline at end of file diff --git a/resources_servers/arena_judge/README.md b/resources_servers/arena_judge/README.md index 83caf6521f..14a8b6865e 100644 --- a/resources_servers/arena_judge/README.md +++ b/resources_servers/arena_judge/README.md @@ -47,16 +47,16 @@ Each JSONL row must carry the following top-level fields (pydantic ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/arena_judge/configs/arena_judge.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server arena_judge # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=arena_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/arena_judge/data/example.jsonl \ - +output_jsonl_fpath=results/arena_judge_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent arena_judge_simple_agent \ + --input resources_servers/arena_judge/data/example.jsonl \ + --output results/arena_judge_rollouts.jsonl \ + --num-repeats 1 ``` ## Configuring the judge diff --git a/resources_servers/asr_with_pc/README.md b/resources_servers/asr_with_pc/README.md index 87430f1115..17e8521715 100644 --- a/resources_servers/asr_with_pc/README.md +++ b/resources_servers/asr_with_pc/README.md @@ -50,19 +50,19 @@ workaround until the schema is extended. ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/asr_with_pc/configs/asr_with_pc.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server asr_with_pc ``` ## Collecting rollouts (5-example smoke test) ```bash -ng_collect_rollouts \ - +agent_name=asr_with_pc_simple_agent \ - +input_jsonl_fpath=resources_servers/asr_with_pc/data/example.jsonl \ - +output_jsonl_fpath=results/asr_with_pc_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent asr_with_pc_simple_agent \ + --input resources_servers/asr_with_pc/data/example.jsonl \ + --output results/asr_with_pc_rollouts.jsonl \ + --num-repeats 1 ``` ## Regenerating example data diff --git a/resources_servers/aviary/README.md b/resources_servers/aviary/README.md index 83307c67de..4f062ac57c 100644 --- a/resources_servers/aviary/README.md +++ b/resources_servers/aviary/README.md @@ -22,17 +22,18 @@ This resources server adapts [Aviary environments](https://github.com/Future-Hou Run the GSM8K Aviary resources server together with a model config: ```bash -config_paths="resources_servers/aviary/configs/gsm8k_aviary.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --resource-server aviary/gsm8k_aviary \ + --model-type vllm_model ``` Then collect rollouts: ```bash -ng_collect_rollouts \ - +agent_name=gsm8k_aviary_agent +input_jsonl_fpath=resources_servers/aviary/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/aviary/data/example_rollouts.jsonl +gym eval run --no-serve \ + --agent gsm8k_aviary_agent \ + --input resources_servers/aviary/data/example.jsonl \ + --output resources_servers/aviary/data/example_rollouts.jsonl ``` # BixBench-Hypothesis (BBH) @@ -48,17 +49,18 @@ Then, prepare your Gym data with the task_idx values of the problems you would l Once the dataset server is running and is accessible at a specific URL, update your config based on [configs/bbh_remote.yaml](configs/bbh_remote.yaml) with the server URL and api key, and launch NeMo-Gym as follows: ```bash -config_paths="resources_servers/aviary/configs/bbh_remote.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --resource-server aviary/bbh_remote \ + --model-type vllm_model ``` -Then collect rollouts on your data as follows (updating the input_jsonl_fpath to your Gym data if needed): +Then collect rollouts on your data as follows (updating the input file to your Gym data if needed): ```bash -ng_collect_rollouts \ - +agent_name=bbh_aviary_agent +input_jsonl_fpath=resources_servers/aviary/data/bbh_train_example.jsonl \ - +output_jsonl_fpath=resources_servers/aviary/data/example_bbh_rollouts.jsonl +gym eval run --no-serve \ + --agent bbh_aviary_agent \ + --input resources_servers/aviary/data/bbh_train_example.jsonl \ + --output resources_servers/aviary/data/example_bbh_rollouts.jsonl ``` To run training with NeMo-RL, set the following fields in your NeMo-RL container (where train_data.jsonl and validation_data.jsonl are set to your train/val Gym data respectively, and bbh_remote.yaml is updated with your dataset server URL/api-key): @@ -98,14 +100,15 @@ cd /path/to/gym/directory ``` And then bring up NeMo-Gym: ```bash -config_paths="resources_servers/aviary/configs/bbh_bundled.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --resource-server aviary/bbh_bundled \ + --model-type vllm_model ``` ```bash -ng_collect_rollouts \ - +agent_name=bbh_aviary_agent +input_jsonl_fpath=resources_servers/aviary/data/bbh_train_example.jsonl \ - +output_jsonl_fpath=resources_servers/aviary/data/example_bbh_rollouts.jsonl +gym eval run --no-serve \ + --agent bbh_aviary_agent \ + --input resources_servers/aviary/data/bbh_train_example.jsonl \ + --output resources_servers/aviary/data/example_bbh_rollouts.jsonl ``` If you are running training with NeMo-RL and NeMo-Gym, add the following modification to your `ray.sub` file in NeMo-RL to support adding a setup command: diff --git a/resources_servers/bigcodebench/README.md b/resources_servers/bigcodebench/README.md index 7a177cd7ef..bd21b9f18e 100644 --- a/resources_servers/bigcodebench/README.md +++ b/resources_servers/bigcodebench/README.md @@ -23,16 +23,16 @@ starts are instant. ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/bigcodebench/configs/bigcodebench.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server bigcodebench # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=bigcodebench_simple_agent \ - +input_jsonl_fpath=resources_servers/bigcodebench/data/example.jsonl \ - +output_jsonl_fpath=results/bigcodebench_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent bigcodebench_simple_agent \ + --input resources_servers/bigcodebench/data/example.jsonl \ + --output results/bigcodebench_rollouts.jsonl \ + --num-repeats 1 ``` ## Reasoning-parser note diff --git a/resources_servers/bird_sql/README.md b/resources_servers/bird_sql/README.md index 8921249e3a..eccb92d2f9 100644 --- a/resources_servers/bird_sql/README.md +++ b/resources_servers/bird_sql/README.md @@ -34,9 +34,9 @@ ensure_bird_sql() ### Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/bird_sql/configs/bird_sql.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server bird_sql ``` Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in @@ -45,11 +45,11 @@ Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in ### Collecting rollouts (5-example smoke test) ```bash -ng_collect_rollouts \ - +agent_name=bird_sql_simple_agent \ - +input_jsonl_fpath=resources_servers/bird_sql/data/example.jsonl \ - +output_jsonl_fpath=results/bird_sql_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent bird_sql_simple_agent \ + --input resources_servers/bird_sql/data/example.jsonl \ + --output results/bird_sql_rollouts.jsonl \ + --num-repeats 1 ``` For a full BIRD dev run, see `benchmarks/birdbench/README.md`. diff --git a/resources_servers/blackjack/README.md b/resources_servers/blackjack/README.md index 54fbff84ad..927f6bc7dc 100644 --- a/resources_servers/blackjack/README.md +++ b/resources_servers/blackjack/README.md @@ -9,7 +9,9 @@ Example data provided in `data/example.jsonl` (system prompt only, no verifier_m ## Run ```bash -ng_run "+config_paths=[resources_servers/blackjack/configs/blackjack.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --resource-server blackjack \ + --model-type vllm_model ``` ## Data @@ -19,8 +21,8 @@ Each game is generated on the fly during `reset()`, so every row in `example.jso ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=blackjack_gymnasium_agent \ - +input_jsonl_fpath=resources_servers/blackjack/data/example.jsonl \ - +output_jsonl_fpath=results/blackjack_rollouts.jsonl +gym eval run --no-serve \ + --agent blackjack_gymnasium_agent \ + --input resources_servers/blackjack/data/example.jsonl \ + --output results/blackjack_rollouts.jsonl ``` diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index 8b2917f32e..f9254c72fa 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -29,10 +29,9 @@ judge_model_name: Qwen/Qwen3-235B-A22B-Instruct-2507 ```bash # If you want to run with browsecomp benchmark instead of the example samples, need to change the datasets part to the one like `benchmarks/browsecomp/config.yaml` in `resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml` -config_paths="resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" - -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --resource-server browsecomp_advanced_harness \ + --model-type vllm_model ``` diff --git a/resources_servers/calendar/README.md b/resources_servers/calendar/README.md index c65fa6a916..c20b933755 100644 --- a/resources_servers/calendar/README.md +++ b/resources_servers/calendar/README.md @@ -16,9 +16,9 @@ The conversations in the dataset are generated using personas from the [nvidia/N The following is an example command for running this resources server along with an OpenAI model: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -resources_servers/calendar/configs/calendar.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --resource-server calendar ``` ## Collecting rollouts @@ -26,11 +26,11 @@ ng_run "+config_paths=[$config_paths]" Rollouts can be collected using the example dataset as follows: ```bash -ng_collect_rollouts \ - +agent_name=calendar_agent \ - +input_jsonl_fpath=data/example.jsonl \ - +output_jsonl_fpath=results/example_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent calendar_agent \ + --input data/example.jsonl \ + --output results/example_rollouts.jsonl \ + --limit 5 ``` The input JSONL file should contain entries with: diff --git a/resources_servers/circle_click/README.md b/resources_servers/circle_click/README.md index 3fcfe8819f..ea751da7f6 100644 --- a/resources_servers/circle_click/README.md +++ b/resources_servers/circle_click/README.md @@ -12,8 +12,14 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & -ng_run "+config_paths=[resources_servers/circle_click/configs/circle_click.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" & -ng_collect_rollouts +agent_name=circle_click_simple_agent +input_jsonl_fpath=resources_servers/circle_click/data/example.jsonl +output_jsonl_fpath=resources_servers/circle_click/data/example_rollouts.jsonl +limit=1 +gym env run \ + --resource-server circle_click \ + --model-type vllm_model & +gym eval run --no-serve \ + --agent circle_click_simple_agent \ + --input resources_servers/circle_click/data/example.jsonl \ + --output resources_servers/circle_click/data/example_rollouts.jsonl \ + --limit 1 ``` # Generating Data diff --git a/resources_servers/circle_count/README.md b/resources_servers/circle_count/README.md index 5d567b04dc..e77665e9bd 100644 --- a/resources_servers/circle_count/README.md +++ b/resources_servers/circle_count/README.md @@ -19,8 +19,14 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & -ng_run "+config_paths=[resources_servers/circle_count/configs/circle_count.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" & -ng_collect_rollouts +agent_name=circle_count_simple_agent +input_jsonl_fpath=resources_servers/circle_count/data/example.jsonl +output_jsonl_fpath=resources_servers/circle_count/data/example_rollouts.jsonl +limit=1 +gym env run \ + --resource-server circle_count \ + --model-type vllm_model & +gym eval run --no-serve \ + --agent circle_count_simple_agent \ + --input resources_servers/circle_count/data/example.jsonl \ + --output resources_servers/circle_count/data/example_rollouts.jsonl \ + --limit 1 ``` # Generating Data diff --git a/resources_servers/code_fim/README.md b/resources_servers/code_fim/README.md index 809548160e..2415e649e9 100644 --- a/resources_servers/code_fim/README.md +++ b/resources_servers/code_fim/README.md @@ -58,16 +58,16 @@ the reward; pass@k / majority@k are computed by the metrics layer. ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/code_fim/configs/code_fim.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server code_fim # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=code_fim_simple_agent \ - +input_jsonl_fpath=resources_servers/code_fim/data/example.jsonl \ - +output_jsonl_fpath=results/code_fim_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent code_fim_simple_agent \ + --input resources_servers/code_fim/data/example.jsonl \ + --output results/code_fim_rollouts.jsonl \ + --num-repeats 1 ``` Start vLLM with `--reasoning-parser ` (e.g. `deepseek_r1` for diff --git a/resources_servers/code_gen/README.md b/resources_servers/code_gen/README.md index bdc2039890..71ec0cd132 100644 --- a/resources_servers/code_gen/README.md +++ b/resources_servers/code_gen/README.md @@ -24,15 +24,16 @@ We use the LiveCodeBench execution code. ```bash # Running the server -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/code_gen/configs/code_gen.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --resource-server code_gen # Collect rollouts from example problems -ng_collect_rollouts +agent_name=code_gen_simple_agent \ - +input_jsonl_fpath=resources_servers/code_gen/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/code_gen/data/example_rollouts.jsonl \ - +limit=null +gym eval run --no-serve \ + --agent code_gen_simple_agent \ + --input resources_servers/code_gen/data/example.jsonl \ + --output resources_servers/code_gen/data/example_rollouts.jsonl \ + --limit null ``` ## Licensing information diff --git a/resources_servers/competitive_coding_challenges/README.md b/resources_servers/competitive_coding_challenges/README.md index ade3fcf957..7dfce341d1 100644 --- a/resources_servers/competitive_coding_challenges/README.md +++ b/resources_servers/competitive_coding_challenges/README.md @@ -61,15 +61,17 @@ elsewhere. Cluster/SLURM users can co-launch the sandbox via Skills' The following allows users to launch the environment followed by the request for generating and collecting rollouts. ```bash -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --config "$config_paths" \ +simple_agent.responses_api_agents.simple_agent.resources_server.name=competitive_coding_challenges_resources_server -ng_collect_rollouts +agent_name=simple_agent \ - +input_jsonl_fpath=$input_jsonl_fpath \ - +output_jsonl_fpath=$output_jsonl_fpath \ - +limit=null \ - +num_repeats=1 \ - +num_samples_in_parallel=null +gym eval run --no-serve \ + --agent simple_agent \ + --input $input_jsonl_fpath \ + --output $output_jsonl_fpath \ + --limit null \ + --num-repeats 1 \ + --concurrency null ``` ## Additional Information diff --git a/resources_servers/cvdp/README.md b/resources_servers/cvdp/README.md index 67db483f47..3251871b55 100644 --- a/resources_servers/cvdp/README.md +++ b/resources_servers/cvdp/README.md @@ -123,7 +123,9 @@ apt install -y ./apptainer_1.3.1_amd64.deb ### Step 1 — Start servers ```bash -ng_run "+config_paths=[resources_servers/cvdp/configs/cvdp.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --resource-server cvdp \ + --model-type vllm_model ``` ### Step 2 — Run rollout collection @@ -131,15 +133,18 @@ ng_run "+config_paths=[resources_servers/cvdp/configs/cvdp.yaml,responses_api_mo `num_repeats` controls how many times each problem is run (for pass@k metrics): ```bash -ng_collect_rollouts \ - +agent_name=cvdp_agent \ - +input_jsonl_fpath=resources_servers/cvdp/data/.jsonl \ - +output_jsonl_fpath=results/rollouts.jsonl \ - +num_repeats=5 \ - +num_samples_in_parallel=4 \ - "+responses_create_params={max_output_tokens: 4096, temperature: 0.2, top_p: 0.7}" \ - "+config_paths=[resources_servers/cvdp/configs/cvdp.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" - "+resume_from_cache=True" +gym eval run --no-serve \ + --agent cvdp_agent \ + --input resources_servers/cvdp/data/.jsonl \ + --output results/rollouts.jsonl \ + --num-repeats 5 \ + --concurrency 4 \ + --max-output-tokens 4096 \ + --temperature 0.2 \ + --top-p 0.7 \ + --resource-server cvdp \ + --model-type vllm_model \ + --resume ``` At the end of this step, you should have rollouts.jsonl, rollouts_agent_metrics.json, rollouts_materialized_inputs.jsonl, rollouts_reward_profiling.jsonl. @@ -158,7 +163,7 @@ python resources_servers/cvdp/scripts/cvdp_pass_at_k_report.py \ | Arg | Required | Default | Description | | ------------ | -------- | ---------- | --------------------------------------------------------- | -| `--rollouts` | Yes | — | Rollout JSONL from `ng_collect_rollouts` | +| `--rollouts` | Yes | — | Rollout JSONL from `gym eval run --no-serve` | | `--output` | Yes | — | Output directory for reports | | `--model` | No | `nemo-gym` | Model name shown in report metadata | | `--dataset` | No | — | Path to original CVDP dataset JSONL (for report metadata) | diff --git a/resources_servers/equivalence_llm_judge/README.md b/resources_servers/equivalence_llm_judge/README.md index f30a1bc582..900a562833 100644 --- a/resources_servers/equivalence_llm_judge/README.md +++ b/resources_servers/equivalence_llm_judge/README.md @@ -51,10 +51,9 @@ equivalence_llm_judge_simple_agent: ### Usage Spin up with a judge model and prompt: ```bash -config_paths="resources_servers/equivalence_llm_judge/configs/equivalence_llm_judge.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --resource-server equivalence_llm_judge \ + --model-type openai_model \ +equivalence_llm_judge.resources_servers.equivalence_llm_judge.judge_responses_create_params.max_output_tokens=256 \ +equivalence_llm_judge.resources_servers.equivalence_llm_judge.judge_system_message="You are a careful arbiter." \ "+equivalence_llm_judge.resources_servers.equivalence_llm_judge.judge_prompt_template='<|Problem|>\n{question}\n\n<|Gold|>\n{expected_answer}\n\n<|Prediction|>\n{generated_answer}\n'" diff --git a/resources_servers/ether0/README.md b/resources_servers/ether0/README.md index f84ab52ccf..c4fab26603 100644 --- a/resources_servers/ether0/README.md +++ b/resources_servers/ether0/README.md @@ -27,13 +27,15 @@ Start servers and collect rollouts ```bash # start vllm and nemo gym servers vllm serve futurehouse/ether0 & -ng_run "+config_paths=[resources_servers/ether0/configs/ether0.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" & +gym env run \ + --resource-server ether0 \ + --model-type vllm_model & # wait for above to be ready -ng_collect_rollouts \ - +agent_name=ether0_simple_agent \ - +input_jsonl_fpath=resources_servers/ether0/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/ether0/data/ether0_rollouts.jsonl +gym eval run --no-serve \ + --agent ether0_simple_agent \ + --input resources_servers/ether0/data/example.jsonl \ + --output resources_servers/ether0/data/ether0_rollouts.jsonl tail -n 1 resources_servers/ether0/data/ether0_rollouts.jsonl | jq | less ``` diff --git a/resources_servers/evalplus/README.md b/resources_servers/evalplus/README.md index d7b6186b00..5024e5c09b 100644 --- a/resources_servers/evalplus/README.md +++ b/resources_servers/evalplus/README.md @@ -46,16 +46,16 @@ compute pass@k for each separately. ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/evalplus/configs/evalplus.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server evalplus # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=evalplus_simple_agent \ - +input_jsonl_fpath=resources_servers/evalplus/data/example.jsonl \ - +output_jsonl_fpath=results/evalplus_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent evalplus_simple_agent \ + --input resources_servers/evalplus/data/example.jsonl \ + --output results/evalplus_rollouts.jsonl \ + --num-repeats 1 ``` Start vLLM with `--reasoning-parser ` (e.g. `deepseek_r1` for diff --git a/resources_servers/example_multi_turn_gymnasium/README.md b/resources_servers/example_multi_turn_gymnasium/README.md index 68d1930157..49cd5bf965 100644 --- a/resources_servers/example_multi_turn_gymnasium/README.md +++ b/resources_servers/example_multi_turn_gymnasium/README.md @@ -15,14 +15,16 @@ Example data provided in `data/example.jsonl`. ## Run ```bash -ng_run "+config_paths=[resources_servers/example_multi_turn_gymnasium/configs/example_multi_turn_gymnasium.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --resource-server example_multi_turn_gymnasium \ + --model-type vllm_model ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=example_multi_turn_gymnasium_agent \ - +input_jsonl_fpath=resources_servers/example_multi_turn_gymnasium/data/example.jsonl \ - +output_jsonl_fpath=results/example_multi_turn_rollouts.jsonl +gym eval run --no-serve \ + --agent example_multi_turn_gymnasium_agent \ + --input resources_servers/example_multi_turn_gymnasium/data/example.jsonl \ + --output results/example_multi_turn_rollouts.jsonl ``` diff --git a/resources_servers/finance_sec_search/README.md b/resources_servers/finance_sec_search/README.md index 6c7afdb064..3518779796 100644 --- a/resources_servers/finance_sec_search/README.md +++ b/resources_servers/finance_sec_search/README.md @@ -152,8 +152,8 @@ public benchmark lives in `benchmarks/finance_sec_search/`. It downloads the `public.csv` dataset from GitHub and converts it to Gym format: ```bash -# Prepare via Gym CLI (recommended — used by ng_run with benchmark configs): -ng_prepare_benchmark +benchmark=benchmarks/finance_sec_search/config_no_web_search.yaml +# Prepare via Gym CLI (recommended — used by gym env run with benchmark configs): +gym eval prepare --benchmark finance_sec_search/config_no_web_search # Or run the script directly: python benchmarks/finance_sec_search/prepare.py # without web_search @@ -194,40 +194,42 @@ Launch a vLLM-compatible model server (e.g. Qwen3-30B-A3B) so the policy and jud With a local vLLM model server: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,resources_servers/finance_sec_search/configs/finance_sec_search.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server finance_sec_search ``` Or with an OpenAI-compatible API (e.g. OpenAI, Azure, NIM): ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,resources_servers/finance_sec_search/configs/finance_sec_search.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --resource-server finance_sec_search ``` ### 4. Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=finance_agent \ - +input_jsonl_fpath=resources_servers/finance_sec_search/data/example.jsonl \ - +output_jsonl_fpath=results/finance_sec_search_rollouts.jsonl +gym eval run --no-serve \ + --agent finance_agent \ + --input resources_servers/finance_sec_search/data/example.jsonl \ + --output results/finance_sec_search_rollouts.jsonl ``` -Add `+limit=1` for a quick single-question test: +Add `--limit 1` for a quick single-question test: ```bash -ng_collect_rollouts \ - +agent_name=finance_agent \ - +input_jsonl_fpath=resources_servers/finance_sec_search/data/example.jsonl \ - +output_jsonl_fpath=results/finance_sec_search_rollouts.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent finance_agent \ + --input resources_servers/finance_sec_search/data/example.jsonl \ + --output results/finance_sec_search_rollouts.jsonl \ + --limit 1 ``` ### Run tests ```bash -ng_test +entrypoint=resources_servers/finance_sec_search +gym env test --resource-server finance_sec_search ``` ## Verification diff --git a/resources_servers/format_verification/README.md b/resources_servers/format_verification/README.md index 978983216f..3ee6df91c0 100644 --- a/resources_servers/format_verification/README.md +++ b/resources_servers/format_verification/README.md @@ -26,63 +26,63 @@ See [ARCHITECTURE.md](ARCHITECTURE.md) for system diagrams and detailed field ma ### Freeform Formatting ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/format_verification/configs/freeform_formatting.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --model-type openai_model \ + --resource-server format_verification/freeform_formatting ``` Collect rollouts: ```bash -ng_collect_rollouts \ - +agent_name=freeform_formatting_simple_agent \ - +input_jsonl_fpath=resources_servers/format_verification/data/ds2_freeform_formatting_train.jsonl \ - +output_jsonl_fpath=results/freeform_formatting_rollouts.jsonl \ - +resume_from_cache=True \ - +num_samples_in_parallel=256 +gym eval run --no-serve \ + --agent freeform_formatting_simple_agent \ + --input resources_servers/format_verification/data/ds2_freeform_formatting_train.jsonl \ + --output results/freeform_formatting_rollouts.jsonl \ + --concurrency 256 \ + --resume ``` ### Citation Format ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/format_verification/configs/citation_format.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --model-type openai_model \ + --resource-server format_verification/citation_format ``` Collect rollouts: ```bash -ng_collect_rollouts \ - +agent_name=citation_format_simple_agent \ - +input_jsonl_fpath=resources_servers/format_verification/data/ds3_citation_format_train.jsonl \ - +output_jsonl_fpath=results/citation_format_rollouts.jsonl \ - +resume_from_cache=True \ - +num_samples_in_parallel=256 +gym eval run --no-serve \ + --agent citation_format_simple_agent \ + --input resources_servers/format_verification/data/ds3_citation_format_train.jsonl \ + --output results/citation_format_rollouts.jsonl \ + --concurrency 256 \ + --resume ``` ## Downloading Data ### Freeform Formatting ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/format_verification/configs/freeform_formatting.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/format_verification_freeform/ \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resource-server format_verification/freeform_formatting \ + --output-dir data/format_verification_freeform/ \ + --mode train_preparation \ + --download ``` ### Citation Format ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/format_verification/configs/citation_format.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/format_verification_citation/ \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resource-server format_verification/citation_format \ + --output-dir data/format_verification_citation/ \ + --mode train_preparation \ + --download ``` ## Testing ```bash -ng_test +entrypoint=resources_servers/format_verification +gym env test --resource-server format_verification ``` ## Licensing diff --git a/resources_servers/frontierscience_judge/README.md b/resources_servers/frontierscience_judge/README.md index a335ff5ddc..b2bb324caf 100644 --- a/resources_servers/frontierscience_judge/README.md +++ b/resources_servers/frontierscience_judge/README.md @@ -24,16 +24,16 @@ judge's full text). ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/frontierscience_judge/configs/frontierscience_judge.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server frontierscience_judge # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=frontierscience_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/frontierscience_judge/data/example.jsonl \ - +output_jsonl_fpath=results/frontierscience_judge_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent frontierscience_judge_simple_agent \ + --input resources_servers/frontierscience_judge/data/example.jsonl \ + --output results/frontierscience_judge_rollouts.jsonl \ + --num-repeats 1 ``` The default `judge_model` config points at the public NVIDIA inference diff --git a/resources_servers/gdpval/README.md b/resources_servers/gdpval/README.md index a82cd07ecd..0295fcd26d 100644 --- a/resources_servers/gdpval/README.md +++ b/resources_servers/gdpval/README.md @@ -13,10 +13,11 @@ Two modes via `reward_mode` config: Canonical entry point is the benchmark at `benchmarks/gdpval/`: ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/gdpval/config.yaml]" -ng_e2e_collect_rollouts \ - "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/gdpval/config.yaml]" \ - ++split=benchmark +gym eval prepare --benchmark gdpval +gym eval run \ + --model-type vllm_model \ + --benchmark gdpval \ + --split benchmark ``` See `benchmarks/gdpval/README.md` for the full run recipe. diff --git a/resources_servers/google_search/README.md b/resources_servers/google_search/README.md index 005fea5343..79c029da86 100644 --- a/resources_servers/google_search/README.md +++ b/resources_servers/google_search/README.md @@ -98,14 +98,15 @@ tools=[ ### Running the Server ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/google_search/configs/google_search.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --resource-server google_search -ng_collect_rollouts +agent_name=simple_agent \ - +input_jsonl_fpath=data/Nemotron-RL-knowledge-web_search-mcqa.jsonl \ - +output_jsonl_fpath=results/Nemotron-RL-knowledge-web_search-mcqa.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent simple_agent \ + --input data/Nemotron-RL-knowledge-web_search-mcqa.jsonl \ + --output results/Nemotron-RL-knowledge-web_search-mcqa.jsonl \ + --limit 1 ``` ## Samples diff --git a/resources_servers/gpqa_diamond/README.md b/resources_servers/gpqa_diamond/README.md index b039946429..9b939d412d 100644 --- a/resources_servers/gpqa_diamond/README.md +++ b/resources_servers/gpqa_diamond/README.md @@ -1,116 +1,117 @@ -# GPQA-Diamond Resources Server - -## Overview - -This resources server evaluates GPQA-Diamond multiple-choice responses with a -GPQA-specific verifier built on top of `resources_servers/mcqa`. - -- Task type: single-turn MCQ -- Domain: `knowledge` -- Dataset prompt format: final line `Answer: LETTER` -- Also accepted by the verifier: `\boxed{X}` and custom regex extraction via - `template_metadata.output_regex` -- Grading mode: `strict_single_letter_boxed` - -## Server Composition - -Use GPQA-Diamond with: - -- `responses_api_agents/simple_agent` -- `responses_api_models/*` (typically `policy_model`) -- `resources_servers/gpqa_diamond` - -The server verifies the model response and returns reward `1.0` for exact -letter match against `expected_answer`, else `0.0`. - -Answer extraction priority is: - -1. `template_metadata.output_regex` if present -2. GPQA fallback parsing of the final answer from `\boxed{...}` or `Answer: X` - -## Dataset Format - -Each JSONL row follows the MCQA request schema: - -- `responses_create_params.input[0].content`: user prompt containing question + options -- `options`: list of letter-to-text maps, e.g. `[{"A": "..."}, {"B": "..."}]` -- `expected_answer`: one of `A/B/C/D` -- `grading_mode`: `strict_single_letter_boxed` -- `template_metadata`: optional per-row metadata, including optional `output_regex` -- `metadata`: passthrough metadata (`explanation`, `subset_for_metrics`, `difficulty`) -- `uuid`: unique row id - -See `data/example.jsonl` for concrete examples. - -Notes: - -- The generated GPQA prompt asks the model to finish with `Answer: LETTER`. -- Although dataset rows keep `grading_mode: strict_single_letter_boxed` for - compatibility with the shared MCQA schema, the GPQA server's custom fallback - parser accepts both `Answer: X` and boxed final answers. - -## Preprocessing Raw GPQA-Diamond - -Full train data is not stored in this repo. - -`dataset_preprocess.py` downloads the GPQA Diamond split from -`Idavidrein/gpqa` on HuggingFace, writes a normalized raw dump, and then writes -the Gym-formatted training set into `data/`. - -From the repository root: - -```bash -python3 resources_servers/gpqa_diamond/dataset_preprocess.py -``` - -This generates: - -- `resources_servers/gpqa_diamond/data/diamond_raw.jsonl` -- `resources_servers/gpqa_diamond/data/train.jsonl` - -`data/example.jsonl` is a curated repo artifact and is not modified by the -preprocess script. There is currently no `validation.jsonl` for this resources -server. - -## Example Usage - -Using a local Nemotron 3 model with `local_vllm_model`: - -```bash -config_paths="responses_api_agents/simple_agent/configs/simple_agent.yaml,responses_api_models/local_vllm_model/configs/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml,resources_servers/gpqa_diamond/configs/gpqa_diamond.yaml" -ng_run "+config_paths=[${config_paths}]" \ - '++policy_model=${inherit_from:NVIDIA-Nemotron-3-Nano-30B-A3B-BF16}' \ - +simple_agent.responses_api_agents.simple_agent.resources_server.name=gpqa_diamond \ - "++NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.responses_api_models.local_vllm_model.vllm_serve_kwargs.mamba_ssm_cache_dtype=float32" \ - "++NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.responses_api_models.local_vllm_model.vllm_serve_kwargs.enable_prefix_caching=False" -``` - -Generic example with `openai_model`: - -```bash -config_paths="responses_api_agents/simple_agent/configs/simple_agent.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/gpqa_diamond/configs/gpqa_diamond.yaml" - -ng_run "+config_paths=[$config_paths]" \ - +simple_agent.responses_api_agents.simple_agent.resources_server.name=gpqa_diamond - -ng_collect_rollouts \ - +agent_name=simple_agent \ - +input_jsonl_fpath=resources_servers/gpqa_diamond/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/gpqa_diamond/data/example_rollouts.jsonl \ - +limit=3 -``` - -`ng_collect_rollouts` also writes sidecar files next to `output_jsonl_fpath`, matching -the same pattern as `test_rollouts*`: - -- `*_materialized_inputs.jsonl` -- `*_reward_profiling.jsonl` -- `*_agent_metrics.json` - -## Licensing - -Code: Apache 2.0 -Configured train dataset license metadata: MIT - +# GPQA-Diamond Resources Server + +## Overview + +This resources server evaluates GPQA-Diamond multiple-choice responses with a +GPQA-specific verifier built on top of `resources_servers/mcqa`. + +- Task type: single-turn MCQ +- Domain: `knowledge` +- Dataset prompt format: final line `Answer: LETTER` +- Also accepted by the verifier: `\boxed{X}` and custom regex extraction via + `template_metadata.output_regex` +- Grading mode: `strict_single_letter_boxed` + +## Server Composition + +Use GPQA-Diamond with: + +- `responses_api_agents/simple_agent` +- `responses_api_models/*` (typically `policy_model`) +- `resources_servers/gpqa_diamond` + +The server verifies the model response and returns reward `1.0` for exact +letter match against `expected_answer`, else `0.0`. + +Answer extraction priority is: + +1. `template_metadata.output_regex` if present +2. GPQA fallback parsing of the final answer from `\boxed{...}` or `Answer: X` + +## Dataset Format + +Each JSONL row follows the MCQA request schema: + +- `responses_create_params.input[0].content`: user prompt containing question + options +- `options`: list of letter-to-text maps, e.g. `[{"A": "..."}, {"B": "..."}]` +- `expected_answer`: one of `A/B/C/D` +- `grading_mode`: `strict_single_letter_boxed` +- `template_metadata`: optional per-row metadata, including optional `output_regex` +- `metadata`: passthrough metadata (`explanation`, `subset_for_metrics`, `difficulty`) +- `uuid`: unique row id + +See `data/example.jsonl` for concrete examples. + +Notes: + +- The generated GPQA prompt asks the model to finish with `Answer: LETTER`. +- Although dataset rows keep `grading_mode: strict_single_letter_boxed` for + compatibility with the shared MCQA schema, the GPQA server's custom fallback + parser accepts both `Answer: X` and boxed final answers. + +## Preprocessing Raw GPQA-Diamond + +Full train data is not stored in this repo. + +`dataset_preprocess.py` downloads the GPQA Diamond split from +`Idavidrein/gpqa` on HuggingFace, writes a normalized raw dump, and then writes +the Gym-formatted training set into `data/`. + +From the repository root: + +```bash +python3 resources_servers/gpqa_diamond/dataset_preprocess.py +``` + +This generates: + +- `resources_servers/gpqa_diamond/data/diamond_raw.jsonl` +- `resources_servers/gpqa_diamond/data/train.jsonl` + +`data/example.jsonl` is a curated repo artifact and is not modified by the +preprocess script. There is currently no `validation.jsonl` for this resources +server. + +## Example Usage + +Using a local Nemotron 3 model with `local_vllm_model`: + +```bash +gym env run \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type local_vllm_model/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 \ + --resource-server gpqa_diamond \ + '++policy_model=${inherit_from:NVIDIA-Nemotron-3-Nano-30B-A3B-BF16}' \ + +simple_agent.responses_api_agents.simple_agent.resources_server.name=gpqa_diamond \ + ++NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.responses_api_models.local_vllm_model.vllm_serve_kwargs.mamba_ssm_cache_dtype=float32 \ + ++NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.responses_api_models.local_vllm_model.vllm_serve_kwargs.enable_prefix_caching=False +``` + +Generic example with `openai_model`: + +```bash +gym env run \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type openai_model \ + --resource-server gpqa_diamond \ + +simple_agent.responses_api_agents.simple_agent.resources_server.name=gpqa_diamond + +gym eval run --no-serve \ + --agent simple_agent \ + --input resources_servers/gpqa_diamond/data/example.jsonl \ + --output resources_servers/gpqa_diamond/data/example_rollouts.jsonl \ + --limit 3 +``` + +`gym eval run` also writes sidecar files next to the `--output` file, matching +the same pattern as `test_rollouts*`: + +- `*_materialized_inputs.jsonl` +- `*_reward_profiling.jsonl` +- `*_agent_metrics.json` + +## Licensing + +Code: Apache 2.0 +Configured train dataset license metadata: MIT + diff --git a/resources_servers/graphwalks/README.md b/resources_servers/graphwalks/README.md index 5f11f4de55..8ccfcfbc46 100644 --- a/resources_servers/graphwalks/README.md +++ b/resources_servers/graphwalks/README.md @@ -25,16 +25,18 @@ https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/evaluation/evaluator ## Start environment ```bash -ng_run "+config_paths=[resources_servers/graphwalks/configs/graphwalks.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --resource-server graphwalks \ + --model-type vllm_model ``` ## Collect example rollouts ```bash -ng_collect_rollouts \ - +agent_name=graphwalks_simple_agent \ - +input_jsonl_fpath=resources_servers/graphwalks/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/graphwalks/data/example_rollouts.jsonl +gym eval run --no-serve \ + --agent graphwalks_simple_agent \ + --input resources_servers/graphwalks/data/example.jsonl \ + --output resources_servers/graphwalks/data/example_rollouts.jsonl ``` For the full benchmark run see diff --git a/resources_servers/grl_sokoban/README.md b/resources_servers/grl_sokoban/README.md index b5ea29628e..38087c2a5f 100644 --- a/resources_servers/grl_sokoban/README.md +++ b/resources_servers/grl_sokoban/README.md @@ -11,19 +11,20 @@ Single-box Sokoban puzzle environment. The environment is implemented under `res Spin up the server alongside a compatible agent: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -responses_api_agents/gymnasium_agent/configs/gymnasium_agent.yaml,\ -resources_servers/grl_sokoban/configs/grl_sokoban.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --config responses_api_agents/gymnasium_agent/configs/gymnasium_agent.yaml \ + --resource-server grl_sokoban ``` Collect trajectories: ```bash -ng_collect_rollouts +agent_name=grl_sokoban_gymnasium_agent \ - +input_jsonl_fpath=resources_servers/grl_sokoban/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/grl_sokoban/data/example_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent grl_sokoban_gymnasium_agent \ + --input resources_servers/grl_sokoban/data/example.jsonl \ + --output resources_servers/grl_sokoban/data/example_rollouts.jsonl \ + --limit 5 ``` ## Data diff --git a/resources_servers/grl_tetris/README.md b/resources_servers/grl_tetris/README.md index a8868397ef..0b1abb283d 100644 --- a/resources_servers/grl_tetris/README.md +++ b/resources_servers/grl_tetris/README.md @@ -12,18 +12,19 @@ GRL Tetris environment in Gymnasium style. The model emits one or more ` Start NeMo Gym servers ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -responses_api_agents/gymnasium_agent/configs/gymnasium_agent.yaml,\ -resources_servers/grl_tetris/configs/grl_tetris.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --config responses_api_agents/gymnasium_agent/configs/gymnasium_agent.yaml \ + --resource-server grl_tetris ``` Collect trajectories: ```bash -ng_collect_rollouts +agent_name=grl_tetris_gymnasium_agent \ - +input_jsonl_fpath=resources_servers/grl_tetris/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/grl_tetris/data/example_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent grl_tetris_gymnasium_agent \ + --input resources_servers/grl_tetris/data/example.jsonl \ + --output resources_servers/grl_tetris/data/example_rollouts.jsonl \ + --limit 5 ``` diff --git a/resources_servers/hotpotqa_qa/README.md b/resources_servers/hotpotqa_qa/README.md index 7ef8d91d00..94a1ad4b51 100644 --- a/resources_servers/hotpotqa_qa/README.md +++ b/resources_servers/hotpotqa_qa/README.md @@ -30,19 +30,19 @@ pass@k for each channel. ## Running servers ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/hotpotqa_qa/configs/hotpotqa_qa.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --resource-server hotpotqa_qa ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=hotpotqa_qa_simple_agent \ - +input_jsonl_fpath=resources_servers/hotpotqa_qa/data/example.jsonl \ - +output_jsonl_fpath=results/hotpotqa_qa_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent hotpotqa_qa_simple_agent \ + --input resources_servers/hotpotqa_qa/data/example.jsonl \ + --output results/hotpotqa_qa_rollouts.jsonl \ + --num-repeats 1 ``` # Licensing information diff --git a/resources_servers/ifbench/README.md b/resources_servers/ifbench/README.md index db95933b68..61f3800bcf 100644 --- a/resources_servers/ifbench/README.md +++ b/resources_servers/ifbench/README.md @@ -45,17 +45,18 @@ This clones the repo, applies patches, and writes the `.installed` marker. On su ## Example Usage ```bash -config_paths="benchmarks/ifbench/config.yaml,responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[$config_paths]" - -ng_collect_rollouts \ - +agent_name=ifbench_benchmark_simple_agent \ - +input_jsonl_fpath=resources_servers/ifbench/data/example.jsonl \ - +prompt_config=benchmarks/ifbench/prompts/default.yaml \ - +output_jsonl_fpath=/tmp/ifbench_rollouts.jsonl \ - +num_repeats=1 \ - "+responses_create_params={max_output_tokens: 4096, temperature: 0.0}" +gym env run \ + --benchmark ifbench \ + --model-type openai_model + +gym eval run --no-serve \ + --agent ifbench_benchmark_simple_agent \ + --input resources_servers/ifbench/data/example.jsonl \ + --output /tmp/ifbench_rollouts.jsonl \ + --num-repeats 1 \ + --max-output-tokens 4096 \ + --temperature 0.0 \ + --prompt-config benchmarks/ifbench/prompts/default.yaml ``` ## Licensing diff --git a/resources_servers/imo_gradingbench/README.md b/resources_servers/imo_gradingbench/README.md index 18a5e910db..05e6bb9ccc 100644 --- a/resources_servers/imo_gradingbench/README.md +++ b/resources_servers/imo_gradingbench/README.md @@ -68,14 +68,14 @@ grade. ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/imo_gradingbench/configs/imo_gradingbench.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server imo_gradingbench # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=imo_gradingbench_simple_agent \ - +input_jsonl_fpath=resources_servers/imo_gradingbench/data/example.jsonl \ - +output_jsonl_fpath=results/imo_gradingbench_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent imo_gradingbench_simple_agent \ + --input resources_servers/imo_gradingbench/data/example.jsonl \ + --output results/imo_gradingbench_rollouts.jsonl \ + --num-repeats 1 ``` diff --git a/resources_servers/imo_proofbench_judge/README.md b/resources_servers/imo_proofbench_judge/README.md index 37ee65a27a..9dab8bfd01 100644 --- a/resources_servers/imo_proofbench_judge/README.md +++ b/resources_servers/imo_proofbench_judge/README.md @@ -22,9 +22,9 @@ Use ``benchmarks/imo_proofbench/`` for the IMO-ProofBench dataset. ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/imo_proofbench_judge/configs/imo_proofbench_judge.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --model-type vllm_model \ + --resource-server imo_proofbench_judge \ +judge_base_url=https://generativelanguage.googleapis.com/v1beta/openai \ "+judge_api_key=$GEMINI_API_KEY" \ +judge_model_name=gemini-2.5-pro @@ -33,11 +33,11 @@ ng_run "+config_paths=[$config_paths]" \ ## Collecting rollouts (5-example smoke test) ```bash -ng_collect_rollouts \ - +agent_name=imo_proofbench_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/imo_proofbench_judge/data/example.jsonl \ - +output_jsonl_fpath=results/imo_proofbench_judge_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent imo_proofbench_judge_simple_agent \ + --input resources_servers/imo_proofbench_judge/data/example.jsonl \ + --output results/imo_proofbench_judge_rollouts.jsonl \ + --num-repeats 1 ``` ## Notes diff --git a/resources_servers/instruction_following/README.md b/resources_servers/instruction_following/README.md index c47f051324..5d266ad526 100644 --- a/resources_servers/instruction_following/README.md +++ b/resources_servers/instruction_following/README.md @@ -47,21 +47,23 @@ The dataset can be found at https://huggingface.co/datasets/nvidia/Nemotron-RL-i ### Usage ```bash -config_paths="responses_api_agents/simple_agent/configs/simple_agent.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/instruction_following/configs/instruction_following.yaml" -ng_run "+config_paths=[$config_paths]" "+simple_agent.responses_api_agents.simple_agent.resources_server.name=instruction_following" - -ng_download_dataset_from_gitlab \ - +dataset_name=instruction_following \ - +version=0.0.1 \ - +artifact_fpath=instruction_following.jsonl \ - +output_fpath=data/instruction_following.jsonl - -ng_collect_rollouts \ - +agent_name=simple_agent \ - +input_jsonl_fpath=data/instruction_following.jsonl \ - +output_jsonl_fpath=data/instruction_following_output.jsonl +limit=5 +gym env run \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type openai_model \ + --resource-server instruction_following \ + +simple_agent.responses_api_agents.simple_agent.resources_server.name=instruction_following + +gym dataset download --storage gitlab \ + --name instruction_following \ + --revision 0.0.1 \ + --artifact instruction_following.jsonl \ + --output data/instruction_following.jsonl + +gym eval run --no-serve \ + --agent simple_agent \ + --input data/instruction_following.jsonl \ + --output data/instruction_following_output.jsonl \ + --limit 5 ``` ### Rollout example @@ -81,7 +83,7 @@ We run reward profiling with Qwen/Qwen3-30B-A3B-Instruct-2507. We evaluate the m ## Testing ```bash -ng_test +entrypoint=resources_servers/instruction_following/ +gym env test --resource-server instruction_following ``` diff --git a/resources_servers/inverse_if/README.md b/resources_servers/inverse_if/README.md index 3bcdf63bcd..a69e0044a5 100644 --- a/resources_servers/inverse_if/README.md +++ b/resources_servers/inverse_if/README.md @@ -6,17 +6,18 @@ Evaluates model responses on the **Inverse IF** (Instruction Following) benchmar ```bash # 1. Run unit tests -ng_test +entrypoint=resources_servers/inverse_if +gym env test --resource-server inverse_if # 2. Start servers (in terminal 1) -config_paths="resources_servers/inverse_if/configs/inverse_if.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --resource-server inverse_if \ + --model-type vllm_model # 3. Collect rollouts on example data (in terminal 2) -ng_collect_rollouts \ - +agent_name=inverse_if_simple_agent \ - +input_jsonl_fpath=resources_servers/inverse_if/data/example.jsonl \ - +output_jsonl_fpath=/tmp/inverse_if_rollouts.jsonl +gym eval run --no-serve \ + --agent inverse_if_simple_agent \ + --input resources_servers/inverse_if/data/example.jsonl \ + --output /tmp/inverse_if_rollouts.jsonl ``` ## Overview @@ -58,10 +59,10 @@ The environment: The `data/example.jsonl` file contains 3 synthetic tasks ready to use: ```bash -ng_collect_rollouts \ - +agent_name=inverse_if_simple_agent \ - +input_jsonl_fpath=resources_servers/inverse_if/data/example.jsonl \ - +output_jsonl_fpath=/tmp/test_rollouts.jsonl +gym eval run --no-serve \ + --agent inverse_if_simple_agent \ + --input resources_servers/inverse_if/data/example.jsonl \ + --output /tmp/test_rollouts.jsonl ``` ### Option B: Full Dataset Setup @@ -88,10 +89,10 @@ ng_collect_rollouts \ 2. **Run on full dataset**: ```bash - ng_collect_rollouts \ - +agent_name=inverse_if_simple_agent \ - +input_jsonl_fpath=resources_servers/inverse_if/data/inverse_if.jsonl \ - +output_jsonl_fpath=/tmp/inverse_if_rollouts.jsonl + gym eval run --no-serve \ + --agent inverse_if_simple_agent \ + --input resources_servers/inverse_if/data/inverse_if.jsonl \ + --output /tmp/inverse_if_rollouts.jsonl ``` ## Testing @@ -100,7 +101,7 @@ ng_collect_rollouts \ ```bash # Run all unit tests -ng_test +entrypoint=resources_servers/inverse_if +gym env test --resource-server inverse_if # Or run directly with pytest cd resources_servers/inverse_if @@ -117,17 +118,18 @@ Tests cover: 1. **Start servers**: ```bash - config_paths="resources_servers/inverse_if/configs/inverse_if.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml" - ng_run "+config_paths=[${config_paths}]" + gym env run \ + --resource-server inverse_if \ + --model-type vllm_model ``` 2. **In another terminal, run on example data**: ```bash - ng_collect_rollouts \ - +agent_name=inverse_if_simple_agent \ - +input_jsonl_fpath=resources_servers/inverse_if/data/example.jsonl \ - +output_jsonl_fpath=/tmp/inverse_if_rollouts.jsonl \ - +limit=3 + gym eval run --no-serve \ + --agent inverse_if_simple_agent \ + --input resources_servers/inverse_if/data/example.jsonl \ + --output /tmp/inverse_if_rollouts.jsonl \ + --limit 3 ``` 3. **View results**: @@ -227,7 +229,7 @@ Each line contains: Key transformations: - Inconsistent rubric keys normalised to `{"id", "criteria"}` - Per-task judge template and system prompt extracted from messages -- `responses_create_params` wraps input for `ng_collect_rollouts` +- `responses_create_params` wraps input for `gym eval run --no-serve` - `model_responses` are discarded ## File Structure diff --git a/resources_servers/inverse_if/data/README.md b/resources_servers/inverse_if/data/README.md index 4e12d4be01..f3c5839142 100644 --- a/resources_servers/inverse_if/data/README.md +++ b/resources_servers/inverse_if/data/README.md @@ -35,10 +35,10 @@ The `example.jsonl` file contains 3 synthetic tasks for quick testing: **Usage:** ```bash -ng_collect_rollouts \ - +agent_name=inverse_if_simple_agent \ - +input_jsonl_fpath=resources_servers/inverse_if/data/example.jsonl \ - +output_jsonl_fpath=/tmp/test_rollouts.jsonl +gym eval run --no-serve \ + --agent inverse_if_simple_agent \ + --input resources_servers/inverse_if/data/example.jsonl \ + --output /tmp/test_rollouts.jsonl ``` ## Raw JSON Format @@ -89,7 +89,7 @@ Each line in the JSONL file: **Key transformations:** - Inconsistent rubric keys normalised to `{"id", "criteria"}` - Per-task judge template and system prompt extracted from messages -- `responses_create_params` wrapper required by `ng_collect_rollouts` +- `responses_create_params` wrapper required by `gym eval run --no-serve` - `model_responses` discarded ## Regenerating JSONL Files diff --git a/resources_servers/jailbreak_detection/README.md b/resources_servers/jailbreak_detection/README.md index 04acbc7f8e..b857e7481d 100644 --- a/resources_servers/jailbreak_detection/README.md +++ b/resources_servers/jailbreak_detection/README.md @@ -27,16 +27,21 @@ The example dataset includes various jailbreak attack patterns: 1. Start the servers: ```bash -ng_run "+config_paths=[resources_servers/jailbreak_detection/configs/jailbreak_detection_nemotron_combined_reward_tp8.yaml,responses_api_models/openai_model/configs/openai_model.yaml,resources_servers/jailbreak_detection/configs/safety_judge_model.yaml]" +gym env run \ + --resource-server jailbreak_detection/jailbreak_detection_nemotron_combined_reward_tp8 \ + --model-type openai_model \ + --resource-server jailbreak_detection/safety_judge_model ``` 2. Collect rollouts: ```bash -ng_collect_rollouts \ - "+config_paths=[resources_servers/jailbreak_detection/configs/jailbreak_detection_nemotron_combined_reward_tp8.yaml,responses_api_models/openai_model/configs/openai_model.yaml,resources_servers/jailbreak_detection/configs/safety_judge_model.yaml]" \ - +agent_name=jailbreak_detection_simple_agent \ - +input_jsonl_fpath=resources_servers/jailbreak_detection/data/example.jsonl \ - +output_jsonl_fpath=results/jailbreak_detection_rollouts.jsonl +gym eval run --no-serve \ + --resource-server jailbreak_detection/jailbreak_detection_nemotron_combined_reward_tp8 \ + --model-type openai_model \ + --resource-server jailbreak_detection/safety_judge_model \ + --agent jailbreak_detection_simple_agent \ + --input resources_servers/jailbreak_detection/data/example.jsonl \ + --output results/jailbreak_detection_rollouts.jsonl ``` ## Configuration diff --git a/resources_servers/labbench2_vlm/README.md b/resources_servers/labbench2_vlm/README.md index 426a8d4cf8..2c2fe07bbc 100644 --- a/resources_servers/labbench2_vlm/README.md +++ b/resources_servers/labbench2_vlm/README.md @@ -128,7 +128,7 @@ the highest-k values for `pass@1[avg-of-{k}]/accuracy` and `pass@{k}/accuracy`. Media files (scientific figures and table images/PDFs) are downloaded from a public GCS bucket by `prepare_data.py`. Validation JSONL can also be downloaded via `gitlab_identifier` (for internal NVIDIA users) using -`ng_prepare_data +should_download=true +data_source=gitlab`. +`gym dataset collate +should_download=true +data_source=gitlab`. Media files must always be downloaded separately via `prepare_data.py` -- the `gitlab_identifier` mechanism only handles the lightweight JSONL files. @@ -178,22 +178,27 @@ putting them in `env.yaml`. Start the servers: ```bash -ng_run "+config_paths=[resources_servers/labbench2_vlm/configs/labbench2_vlm.yaml,resources_servers/labbench2_vlm/configs/judge_model_openai.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env run \ + --resource-server labbench2_vlm \ + --resource-server labbench2_vlm/judge_model_openai \ + --model-type openai_model ``` Collect rollouts: ```bash -ng_collect_rollouts \ - +agent_name=labbench2_vlm_simple_agent \ - "+config_paths=[resources_servers/labbench2_vlm/configs/labbench2_vlm.yaml,resources_servers/labbench2_vlm/configs/judge_model_openai.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - +input_jsonl_fpath=resources_servers/labbench2_vlm/data/figqa2_img_validation.jsonl \ - +output_jsonl_fpath=results/figqa2_img_rollouts.jsonl \ - +num_repeats=1 \ +gym eval run --no-serve \ + --resource-server labbench2_vlm \ + --resource-server labbench2_vlm/judge_model_openai \ + --model-type openai_model \ + --agent labbench2_vlm_simple_agent \ + --input resources_servers/labbench2_vlm/data/figqa2_img_validation.jsonl \ + --output results/figqa2_img_rollouts.jsonl \ + --num-repeats 1 \ "+responses_create_params={max_output_tokens: 2048, reasoning: {effort: high}}" ``` -`ng_collect_rollouts` writes sidecar files next to `output_jsonl_fpath`: +`gym eval run --no-serve` writes sidecar files next to the `--output` file: - `*_materialized_inputs.jsonl` - `*_aggregate_metrics.json` @@ -202,12 +207,14 @@ Run the example data for a quick smoke test (works immediately after `git clone` because `data/test_media/` and `data/example.jsonl` are committed): ```bash -ng_collect_rollouts \ - +agent_name=labbench2_vlm_simple_agent \ - "+config_paths=[resources_servers/labbench2_vlm/configs/labbench2_vlm.yaml,resources_servers/labbench2_vlm/configs/judge_model_openai.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - +input_jsonl_fpath=resources_servers/labbench2_vlm/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/labbench2_vlm/data/example_rollouts.jsonl \ - +num_repeats=1 \ +gym eval run --no-serve \ + --resource-server labbench2_vlm \ + --resource-server labbench2_vlm/judge_model_openai \ + --model-type openai_model \ + --agent labbench2_vlm_simple_agent \ + --input resources_servers/labbench2_vlm/data/example.jsonl \ + --output resources_servers/labbench2_vlm/data/example_rollouts.jsonl \ + --num-repeats 1 \ "+responses_create_params={max_output_tokens: 2048, reasoning: {effort: high}}" ``` @@ -216,20 +223,22 @@ rollout output). Regenerate it locally with the command above. ### One-shot alternative -`ng_e2e_collect_rollouts` starts the server stack, preprocesses, and collects -rollouts in a single command (don't run `ng_run` separately). Input path and +`gym eval run` starts the server stack, preprocesses, and collects +rollouts in a single command (don't run `gym env run` separately). Input path and agent ref are auto-derived from the dataset entry in the chained config -(`++split` picks which one): +(`--split` picks which one): ```bash -ng_e2e_collect_rollouts \ - "+config_paths=[resources_servers/labbench2_vlm/configs/labbench2_vlm.yaml,resources_servers/labbench2_vlm/configs/judge_model_openai.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - ++split=validation \ - ++output_jsonl_fpath=results/labbench2_vlm_validation.jsonl \ - +num_samples_in_parallel=16 +gym eval run \ + --resource-server labbench2_vlm \ + --resource-server labbench2_vlm/judge_model_openai \ + --model-type openai_model \ + --split validation \ + --output results/labbench2_vlm_validation.jsonl \ + --concurrency 16 ``` -For a fast smoke test, add `+limit=10 +num_repeats=1`. +For a fast smoke test, add `--limit 10 --num-repeats 1`. ## Throttling @@ -238,10 +247,10 @@ endpoints see roughly `2 × num_samples_in_parallel` concurrent requests. On a hosted endpoint you'll likely hit rate limits or socket errors (`Hit N global ClientOSError`) well before saturating your machine. -Cap concurrency with `+num_samples_in_parallel=`: +Cap concurrency with `--concurrency `: ```bash -ng_collect_rollouts ... +num_samples_in_parallel=16 +gym eval run --no-serve ... --concurrency 16 ``` Start around 16 and bump up if it holds. diff --git a/resources_servers/longmt_eval/README.md b/resources_servers/longmt_eval/README.md index de5584b38e..3bf4e73582 100644 --- a/resources_servers/longmt_eval/README.md +++ b/resources_servers/longmt_eval/README.md @@ -145,9 +145,9 @@ and output rows. ### Reward profiling from pre-baked rollouts (no GPU, no model server) ```bash -ng_reward_profile \ - +materialized_inputs_jsonl_fpath=resources_servers/longmt_eval/data/example_rollouts_materialized_inputs.jsonl \ - +rollouts_jsonl_fpath=resources_servers/longmt_eval/data/example_rollouts.jsonl +gym eval profile \ + --inputs resources_servers/longmt_eval/data/example_rollouts_materialized_inputs.jsonl \ + --rollouts resources_servers/longmt_eval/data/example_rollouts.jsonl ``` Outputs `example_rollouts_reward_profiling.jsonl` (per-task stats) and @@ -158,20 +158,22 @@ rollouts file. ```bash # Start servers (smoke-test mode — no GPU needed for the verifier) -ng_run "+config_paths=[resources_servers/longmt_eval/configs/longmt_eval.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - "++longmt_eval.resources_servers.longmt_eval.compute_segale=false" & +gym env run \ + --resource-server longmt_eval \ + --model-type vllm_model & \ + ++longmt_eval.resources_servers.longmt_eval.compute_segale=false # Collect rollouts — also writes results/longmt_eval_rollouts_materialized_inputs.jsonl -ng_collect_rollouts \ - +agent_name=longmt_eval_simple_agent \ - +input_jsonl_fpath=resources_servers/longmt_eval/data/example.jsonl \ - +output_jsonl_fpath=results/longmt_eval_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent longmt_eval_simple_agent \ + --input resources_servers/longmt_eval/data/example.jsonl \ + --output results/longmt_eval_rollouts.jsonl \ + --num-repeats 1 # Profile rewards from the collected rollouts -ng_reward_profile \ - +materialized_inputs_jsonl_fpath=results/longmt_eval_rollouts_materialized_inputs.jsonl \ - +rollouts_jsonl_fpath=results/longmt_eval_rollouts.jsonl +gym eval profile \ + --inputs results/longmt_eval_rollouts_materialized_inputs.jsonl \ + --rollouts results/longmt_eval_rollouts.jsonl ``` For a full SLURM run with SEGALE enabled on WMT24++ see diff --git a/resources_servers/math_advanced_calculations/README.md b/resources_servers/math_advanced_calculations/README.md index 2aa2601e90..eed8c1581a 100644 --- a/resources_servers/math_advanced_calculations/README.md +++ b/resources_servers/math_advanced_calculations/README.md @@ -8,18 +8,18 @@ Commands - Spin up server: ``` - config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/math_advanced_calculations/configs/math_advanced_calculations.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --resource-server math_advanced_calculations ``` Collect trajectories: ``` -ng_collect_rollouts +agent_name=math_advanced_calculations_simple_agent \ - +input_jsonl_fpath=resources_servers/math_advanced_calculations/data/train.jsonl \ - +output_jsonl_fpath=results/math_advanced_calculations_trajectory_collection.jso -nl \ - +limit=1 +gym eval run --no-serve \ + --agent math_advanced_calculations_simple_agent \ + --input resources_servers/math_advanced_calculations/data/train.jsonl \ + --output results/math_advanced_calculations_trajectory_collection.jsonl \ + --limit 1 ``` Data links: https://huggingface.co/datasets/nvidia/Nemotron-RL-math-advanced_calculations diff --git a/resources_servers/math_proof_judgement/README.md b/resources_servers/math_proof_judgement/README.md index 3959e44044..10d8a9183a 100644 --- a/resources_servers/math_proof_judgement/README.md +++ b/resources_servers/math_proof_judgement/README.md @@ -66,14 +66,14 @@ parser name that matches your model's reasoning tokens (see `vllm serve ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/math_proof_judgement/configs/math_proof_judgement.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server math_proof_judgement # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=math_proof_judgement_simple_agent \ - +input_jsonl_fpath=resources_servers/math_proof_judgement/data/example.jsonl \ - +output_jsonl_fpath=results/math_proof_judgement_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent math_proof_judgement_simple_agent \ + --input resources_servers/math_proof_judgement/data/example.jsonl \ + --output results/math_proof_judgement_rollouts.jsonl \ + --num-repeats 1 ``` diff --git a/resources_servers/math_with_autograder/README.md b/resources_servers/math_with_autograder/README.md index 100c400146..d842b3818e 100644 --- a/resources_servers/math_with_autograder/README.md +++ b/resources_servers/math_with_autograder/README.md @@ -35,8 +35,10 @@ existing math_with_judge consumers. ## Run servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,resources_servers/math_with_autograder/configs/math_with_autograder.yaml,resources_servers/math_with_autograder/configs/judge_gptoss20b.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server math_with_autograder \ + --resource-server math_with_autograder/judge_gptoss20b ``` The bundled `judge_gptoss20b.yaml` wires the autograder judge to @@ -47,11 +49,11 @@ swap to a different OpenAI-compatible endpoint. ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=math_with_autograder_simple_agent \ - +input_jsonl_fpath=resources_servers/math_with_autograder/data/example.jsonl \ - +output_jsonl_fpath=results/math_with_autograder_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent math_with_autograder_simple_agent \ + --input resources_servers/math_with_autograder/data/example.jsonl \ + --output results/math_with_autograder_rollouts.jsonl \ + --num-repeats 1 ``` ## Metrics diff --git a/resources_servers/math_with_code/README.md b/resources_servers/math_with_code/README.md index b3a43a2a98..17f4d43dd4 100644 --- a/resources_servers/math_with_code/README.md +++ b/resources_servers/math_with_code/README.md @@ -95,23 +95,31 @@ OR If you want to download directly ``` -ng_download_dataset_from_gitlab \ - +dataset_name=open_math_reasoning_problems_tool \ - +version=0.0.1 \ - +artifact_fpath=open_math_reasoning_problems_tool.jsonl \ - +output_fpath=data/open_math_reasoning_problems_tool.jsonl +gym dataset download --storage gitlab \ + --name open_math_reasoning_problems_tool \ + --revision 0.0.1 \ + --artifact open_math_reasoning_problems_tool.jsonl \ + --output data/open_math_reasoning_problems_tool.jsonl ``` Start server ``` -ng_run "+config_paths=[responses_api_agents/simple_agent/configs/simple_agent.yaml,responses_api_models/openai_model/configs/openai_model.yaml,resources_servers/math_with_code/configs/math_with_code.yaml]" +simple_agent.responses_api_agents.simple_agent.resources_server.name=math_with_code +gym env run \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type openai_model \ + --resource-server math_with_code \ + +simple_agent.responses_api_agents.simple_agent.resources_server.name=math_with_code ``` Collect trajectories ``` -ng_collect_rollouts +agent_name=simple_agent +input_jsonl_fpath=data/open_math_reasoning_problems_tool.jsonl +output_jsonl_fpath=results/open_math_reasoning_problems_tool_output_new.jsonl +limit=1 +gym eval run --no-serve \ + --agent simple_agent \ + --input data/open_math_reasoning_problems_tool.jsonl \ + --output results/open_math_reasoning_problems_tool_output_new.jsonl \ + --limit 1 ``` diff --git a/resources_servers/math_with_judge/README.md b/resources_servers/math_with_judge/README.md index 0fdfe0318b..46366149bc 100644 --- a/resources_servers/math_with_judge/README.md +++ b/resources_servers/math_with_judge/README.md @@ -11,38 +11,38 @@ on Hugging Face. ## Running servers The following are example commands for running this resources server, along with the simple agent and an OpenAI model: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -resources_servers/math_with_judge/configs/math_with_judge.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --model-type openai_model \ + --resource-server math_with_judge \ +math_with_judge.resources_servers.math_with_judge.judge_model_server.name=policy_model ``` To download the OpenMathReasoning dataset, the following command can be run: ```bash -ng_download_dataset_from_gitlab \ - +dataset_name=math_open_math_reasoning \ - +version=0.0.1 \ - +artifact_fpath=open_math_reasoning_problems.jsonl \ - +output_fpath=data/open_math_reasoning_problems.jsonl +gym dataset download --storage gitlab \ + --name math_open_math_reasoning \ + --revision 0.0.1 \ + --artifact open_math_reasoning_problems.jsonl \ + --output data/open_math_reasoning_problems.jsonl ``` Then, rollouts can be collected using a command such as the following: ```bash -ng_collect_rollouts \ - +agent_name=math_with_judge_simple_agent \ - +input_jsonl_fpath=data/open_math_reasoning_problems.jsonl \ - +output_jsonl_fpath=results/example_open_math_reasoning_verify_responses.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent math_with_judge_simple_agent \ + --input data/open_math_reasoning_problems.jsonl \ + --output results/example_open_math_reasoning_verify_responses.jsonl \ + --limit 5 ``` ## Prepare for trajectory collection ```bash -config_paths="resources_servers/math_with_judge/configs/dapo17k_trajectory_collection.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_prepare_data "+config_paths=[$config_paths]" \ - +output_dirpath=data/dapo17k_trajectory_collection \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config resources_servers/math_with_judge/configs/dapo17k_trajectory_collection.yaml \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/dapo17k_trajectory_collection \ + --mode train_preparation \ + --download ``` # Licensing information diff --git a/resources_servers/mcqa/README.md b/resources_servers/mcqa/README.md index 4fdb3439ef..68f4c4be41 100644 --- a/resources_servers/mcqa/README.md +++ b/resources_servers/mcqa/README.md @@ -111,28 +111,27 @@ For datasets with custom prompt formats, you can optionally use `template_metada **Standard format (with `grading_mode`):** ```bash -config_paths="responses_api_agents/simple_agent/configs/simple_agent.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/mcqa/configs/mcqa.yaml" - - -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type openai_model \ + --resource-server mcqa \ +simple_agent.responses_api_agents.simple_agent.resources_server.name=mcqa -ng_collect_rollouts \ - +agent_name=simple_agent \ - +input_jsonl_fpath=data/MCQA_filtered_decontaminated.jsonl \ - +output_jsonl_fpath=data/MCQA_filtered_decontaminated_samples_rollouts.jsonl +limit=5 +gym eval run --no-serve \ + --agent simple_agent \ + --input data/MCQA_filtered_decontaminated.jsonl \ + --output data/MCQA_filtered_decontaminated_samples_rollouts.jsonl \ + --limit 5 ``` **With template_metadata (custom regex):** ```bash # Using example file with 5 different custom prompt formats -ng_collect_rollouts \ - +agent_name=simple_agent \ - +input_jsonl_fpath=resources_servers/mcqa/data/example_with_template_metadata.jsonl \ - +output_jsonl_fpath=resources_servers/mcqa/data/example_rollouts_with_template_metadata.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent simple_agent \ + --input resources_servers/mcqa/data/example_with_template_metadata.jsonl \ + --output resources_servers/mcqa/data/example_rollouts_with_template_metadata.jsonl \ + --limit 5 ``` Rollout example diff --git a/resources_servers/mrcr/README.md b/resources_servers/mrcr/README.md index d067f11789..093fe529ff 100644 --- a/resources_servers/mrcr/README.md +++ b/resources_servers/mrcr/README.md @@ -27,16 +27,18 @@ always fails. ## Start environment ```bash -ng_run "+config_paths=[resources_servers/mrcr/configs/mrcr.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --resource-server mrcr \ + --model-type vllm_model ``` ## Collect example rollouts ```bash -ng_collect_rollouts \ - +agent_name=mrcr_simple_agent \ - +input_jsonl_fpath=resources_servers/mrcr/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/mrcr/data/example_rollouts.jsonl +gym eval run --no-serve \ + --agent mrcr_simple_agent \ + --input resources_servers/mrcr/data/example.jsonl \ + --output resources_servers/mrcr/data/example_rollouts.jsonl ``` For the full benchmark run see diff --git a/resources_servers/multichallenge/README.md b/resources_servers/multichallenge/README.md index 6e877d9421..7c9a4465ad 100644 --- a/resources_servers/multichallenge/README.md +++ b/resources_servers/multichallenge/README.md @@ -6,17 +6,18 @@ Evaluates model responses on the **MultiChallenge** benchmark using an LLM judge ```bash # 1. Run unit tests -ng_test +entrypoint=resources_servers/multichallenge +gym env test --resource-server multichallenge # 2. Start servers (in terminal 1) -config_paths="resources_servers/multichallenge/configs/multichallenge.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --resource-server multichallenge \ + --model-type vllm_model # 3. Collect rollouts on example data (in terminal 2) -ng_collect_rollouts \ - +agent_name=multichallenge_simple_agent \ - +input_jsonl_fpath=resources_servers/multichallenge/data/example.jsonl \ - +output_jsonl_fpath=/tmp/multichallenge_rollouts.jsonl +gym eval run --no-serve \ + --agent multichallenge_simple_agent \ + --input resources_servers/multichallenge/data/example.jsonl \ + --output /tmp/multichallenge_rollouts.jsonl ``` ## Overview @@ -40,10 +41,10 @@ The `data/example.jsonl` file contains 3 synthetic tasks ready to use: ```bash # No preprocessing needed - just run -ng_collect_rollouts \ - +agent_name=multichallenge_simple_agent \ - +input_jsonl_fpath=resources_servers/multichallenge/data/example.jsonl \ - +output_jsonl_fpath=/tmp/test_rollouts.jsonl +gym eval run --no-serve \ + --agent multichallenge_simple_agent \ + --input resources_servers/multichallenge/data/example.jsonl \ + --output /tmp/test_rollouts.jsonl ``` ### Option B: Full Dataset Setup @@ -76,10 +77,10 @@ ng_collect_rollouts \ 2. **Run on full dataset**: ```bash - ng_collect_rollouts \ - +agent_name=multichallenge_simple_agent \ - +input_jsonl_fpath=resources_servers/multichallenge/data/advanced.jsonl \ - +output_jsonl_fpath=/tmp/advanced_rollouts.jsonl + gym eval run --no-serve \ + --agent multichallenge_simple_agent \ + --input resources_servers/multichallenge/data/advanced.jsonl \ + --output /tmp/advanced_rollouts.jsonl ``` ## Testing @@ -88,7 +89,7 @@ ng_collect_rollouts \ ```bash # Run all unit tests -ng_test +entrypoint=resources_servers/multichallenge +gym env test --resource-server multichallenge # Or run directly with pytest for more detail cd resources_servers/multichallenge @@ -105,17 +106,18 @@ Tests cover: 1. **Start servers**: ```bash - config_paths="resources_servers/multichallenge/configs/multichallenge.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml" - ng_run "+config_paths=[${config_paths}]" + gym env run \ + --resource-server multichallenge \ + --model-type vllm_model ``` 2. **In another terminal, run on example data**: ```bash - ng_collect_rollouts \ - +agent_name=multichallenge_simple_agent \ - +input_jsonl_fpath=resources_servers/multichallenge/data/example.jsonl \ - +output_jsonl_fpath=/tmp/multichallenge_rollouts.jsonl \ - +limit=3 + gym eval run --no-serve \ + --agent multichallenge_simple_agent \ + --input resources_servers/multichallenge/data/example.jsonl \ + --output /tmp/multichallenge_rollouts.jsonl \ + --limit 3 ``` 3. **View results**: @@ -240,7 +242,7 @@ Each line contains: Key transformations: - `thinking` role messages are excluded from input - `context` is pre-formatted for the LLM judge -- `responses_create_params` wraps input for `ng_collect_rollouts` +- `responses_create_params` wraps input for `gym eval run` ## File Structure diff --git a/resources_servers/multichallenge/data/README.md b/resources_servers/multichallenge/data/README.md index d8db78eb6d..9517579b31 100644 --- a/resources_servers/multichallenge/data/README.md +++ b/resources_servers/multichallenge/data/README.md @@ -45,10 +45,10 @@ The `example.jsonl` file contains 3 synthetic tasks for quick testing: **Usage:** ```bash -ng_collect_rollouts \ - +agent_name=multichallenge_simple_agent \ - +input_jsonl_fpath=resources_servers/multichallenge/data/example.jsonl \ - +output_jsonl_fpath=/tmp/test_rollouts.jsonl +gym eval run --no-serve \ + --agent multichallenge_simple_agent \ + --input resources_servers/multichallenge/data/example.jsonl \ + --output /tmp/test_rollouts.jsonl ``` ## Raw JSON Format @@ -106,7 +106,7 @@ Each line in the JSONL file: **Key transformations:** - `thinking` role messages are **excluded** from `responses_create_params.input` - `context` is a pre-formatted string for the LLM judge (also excludes thinking) -- `responses_create_params` wrapper is required by `ng_collect_rollouts` +- `responses_create_params` wrapper is required by `gym eval run` - `metadata` preserves full original data for reference ## Regenerating JSONL Files diff --git a/resources_servers/newton_bench/README.md b/resources_servers/newton_bench/README.md index 75e6749a72..c8359dc7a9 100644 --- a/resources_servers/newton_bench/README.md +++ b/resources_servers/newton_bench/README.md @@ -91,22 +91,23 @@ vllm serve \ ### Launch servers ```bash -config_paths="resources_servers/newton_bench/configs/newton_bench.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --resource-server newton_bench \ + --model-type vllm_model ``` ### Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=newton_bench_simple_agent \ - +input_jsonl_fpath=resources_servers/newton_bench/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/newton_bench/data/example_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent newton_bench_simple_agent \ + --input resources_servers/newton_bench/data/example.jsonl \ + --output resources_servers/newton_bench/data/example_rollouts.jsonl \ + --limit 5 ``` ## Running Tests ```bash -ng_test +entrypoint=resources_servers/newton_bench +gym env test --resource-server newton_bench ``` ## Qwen/Qwen3-VL-8B-Thinking Evaluation Summary diff --git a/resources_servers/ns_tools/README.md b/resources_servers/ns_tools/README.md index 571b764e92..b2137d933e 100644 --- a/resources_servers/ns_tools/README.md +++ b/resources_servers/ns_tools/README.md @@ -8,23 +8,23 @@ It integrates with the NeMo Skills ToolManager to dynamically load and execute t ## Running servers The following are example commands for running this resources server with the simple agent and a vLLM model: ```bash -config_paths="resources_servers/ns_tools/configs/ns_tools.yaml, \ -resources_servers/math_with_judge/configs/math_with_judge.yaml, \ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[$config_paths]" \ - +policy_base_url="http://localhost:8000/v1" \ - +policy_model_name="Qwen/Qwen3-8B" \ +gym env run \ + --resource-server ns_tools \ + --resource-server math_with_judge \ + --model-type vllm_model \ + --model-url http://localhost:8000/v1 \ + --model Qwen/Qwen3-8B \ ++math_with_judge.resources_servers.math_with_judge.should_use_judge=false \ ++math_with_judge.resources_servers.math_with_judge.judge_model_server.name=policy_model ``` Then, rollouts can be collected using a command such as the following: ```bash -ng_collect_rollouts \ - +agent_name=ns_tools_simple_agent \ - +input_jsonl_fpath=resources_servers/ns_tools/data/example.jsonl \ - +output_jsonl_fpath=data/ns_tools_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent ns_tools_simple_agent \ + --input resources_servers/ns_tools/data/example.jsonl \ + --output data/ns_tools_rollouts.jsonl \ + --limit 5 ``` ## Sample data format @@ -78,9 +78,11 @@ python prepare_dataset.py \ To validate the example data and regenerate metrics: ```bash -ng_prepare_data "+config_paths=[resources_servers/ns_tools/configs/ns_tools.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - +output_dirpath=data/ns_tools \ - +mode=example_validation +gym dataset collate \ + --resource-server ns_tools \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/ns_tools \ + --mode example_validation ``` # Licensing information diff --git a/resources_servers/openenv/README.md b/resources_servers/openenv/README.md index d56f9d4026..22032cfba4 100644 --- a/resources_servers/openenv/README.md +++ b/resources_servers/openenv/README.md @@ -36,23 +36,25 @@ policy_model_name: Running an environment is a **two-step process**: -1. **Start the servers** with `ng_run` — this launches the resource server, agent server, and model server. Wait until you see `All 3 / 3 servers ready!`. -2. **Collect rollouts** with `ng_collect_rollouts` in a **separate terminal** — this sends the JSONL prompts through the agent, which calls the LLM and environment in a loop, and writes trajectories with rewards to an output file. +1. **Start the servers** with `gym env run` — this launches the resource server, agent server, and model server. Wait until you see `All 3 / 3 servers ready!`. +2. **Collect rollouts** with `gym eval run --no-serve` in a **separate terminal** — this sends the JSONL prompts through the agent, which calls the LLM and environment in a loop, and writes trajectories with rewards to an output file. #### Echo ```bash # Terminal 1: Start servers source .venv/bin/activate -ng_run "+config_paths=[resources_servers/openenv/configs/openenv_echo.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env run \ + --resource-server openenv/openenv_echo \ + --model-type openai_model # Terminal 2: Collect rollouts (after "All 3 / 3 servers ready!") source .venv/bin/activate -ng_collect_rollouts \ - +agent_name=openenv_echo_simple_agent \ - +input_jsonl_fpath=resources_servers/openenv/data/echo/example.jsonl \ - +output_jsonl_fpath=output_echo.jsonl \ - +num_samples_in_parallel=5 +gym eval run --no-serve \ + --agent openenv_echo_simple_agent \ + --input resources_servers/openenv/data/echo/example.jsonl \ + --output output_echo.jsonl \ + --concurrency 5 ``` #### Coding @@ -60,15 +62,17 @@ ng_collect_rollouts \ ```bash # Terminal 1: Start servers source .venv/bin/activate -ng_run "+config_paths=[resources_servers/openenv/configs/openenv_coding.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env run \ + --resource-server openenv/openenv_coding \ + --model-type openai_model # Terminal 2: Collect rollouts source .venv/bin/activate -ng_collect_rollouts \ - +agent_name=openenv_coding_simple_agent \ - +input_jsonl_fpath=resources_servers/openenv/data/coding/example.jsonl \ - +output_jsonl_fpath=output_coding.jsonl \ - +num_samples_in_parallel=5 +gym eval run --no-serve \ + --agent openenv_coding_simple_agent \ + --input resources_servers/openenv/data/coding/example.jsonl \ + --output output_coding.jsonl \ + --concurrency 5 ``` #### Maze @@ -76,15 +80,17 @@ ng_collect_rollouts \ ```bash # Terminal 1: Start servers source .venv/bin/activate -ng_run "+config_paths=[resources_servers/openenv/configs/openenv_maze.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env run \ + --resource-server openenv/openenv_maze \ + --model-type openai_model # Terminal 2: Collect rollouts source .venv/bin/activate -ng_collect_rollouts \ - +agent_name=openenv_maze_simple_agent \ - +input_jsonl_fpath=resources_servers/openenv/data/maze/example.jsonl \ - +output_jsonl_fpath=output_maze.jsonl \ - +num_samples_in_parallel=5 +gym eval run --no-serve \ + --agent openenv_maze_simple_agent \ + --input resources_servers/openenv/data/maze/example.jsonl \ + --output output_maze.jsonl \ + --concurrency 5 ``` ## Adding a New Environment @@ -178,15 +184,17 @@ Then start servers and collect rollouts: ```bash # Terminal 1: Start servers source .venv/bin/activate -ng_run "+config_paths=[resources_servers/openenv/configs/openenv_.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env run \ + --config resources_servers/openenv/configs/openenv_.yaml \ + --model-type openai_model # Terminal 2: Collect rollouts (after servers are ready) source .venv/bin/activate -ng_collect_rollouts \ - +agent_name=openenv__simple_agent \ - +input_jsonl_fpath=resources_servers/openenv/data//example.jsonl \ - +output_jsonl_fpath=output_.jsonl \ - +num_samples_in_parallel=5 +gym eval run --no-serve \ + --agent openenv__simple_agent \ + --input resources_servers/openenv/data//example.jsonl \ + --output output_.jsonl \ + --concurrency 5 ``` ## How It Works diff --git a/resources_servers/physics_judge/README.md b/resources_servers/physics_judge/README.md index 9444dbe3ed..8cec2511f9 100644 --- a/resources_servers/physics_judge/README.md +++ b/resources_servers/physics_judge/README.md @@ -32,10 +32,10 @@ existing math_with_judge consumers (`aime24`, `aime25`, `gsm8k`, ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/physics_judge/configs/physics_judge.yaml,\ -resources_servers/physics_judge/configs/judge_openai.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server physics_judge \ + --resource-server physics_judge/judge_openai ``` The bundled `judge_openai.yaml` defaults the judge to `openai/gpt-oss-20b` @@ -53,11 +53,11 @@ CLI, or replace `judge_openai.yaml` with your own config that defines a ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=physics_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/physics_judge/data/example.jsonl \ - +output_jsonl_fpath=results/physics_judge_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent physics_judge_simple_agent \ + --input resources_servers/physics_judge/data/example.jsonl \ + --output results/physics_judge_rollouts.jsonl \ + --num-repeats 1 ``` ## Metrics diff --git a/resources_servers/polymath/README.md b/resources_servers/polymath/README.md index 8bae6bd138..17a20bb0f1 100644 --- a/resources_servers/polymath/README.md +++ b/resources_servers/polymath/README.md @@ -30,19 +30,19 @@ PolyMath additions are at the metric-aggregation layer: ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/polymath/configs/polymath.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server polymath ``` ## Collecting rollouts (5-example smoke test) ```bash -ng_collect_rollouts \ - +agent_name=polymath_simple_agent \ - +input_jsonl_fpath=resources_servers/polymath/data/example.jsonl \ - +output_jsonl_fpath=results/polymath_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent polymath_simple_agent \ + --input resources_servers/polymath/data/example.jsonl \ + --output results/polymath_rollouts.jsonl \ + --num-repeats 1 ``` The server's vLLM endpoint should be started with diff --git a/resources_servers/rdkit_chemistry/README.md b/resources_servers/rdkit_chemistry/README.md index 6b7e87656a..3e4c453143 100644 --- a/resources_servers/rdkit_chemistry/README.md +++ b/resources_servers/rdkit_chemistry/README.md @@ -56,15 +56,14 @@ See `data/example.jsonl` for concrete examples. ## Example Usage ```bash -config_paths="resources_servers/rdkit_chemistry/configs/rdkit_chemistry.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[${config_paths}]" - -ng_collect_rollouts \ - +agent_name=rdkit_chemistry_agent \ - +input_jsonl_fpath=resources_servers/rdkit_chemistry/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/rdkit_chemistry/data/example_rollouts.jsonl +gym env run \ + --resource-server rdkit_chemistry \ + --model-type openai_model + +gym eval run --no-serve \ + --agent rdkit_chemistry_agent \ + --input resources_servers/rdkit_chemistry/data/example.jsonl \ + --output resources_servers/rdkit_chemistry/data/example_rollouts.jsonl ``` ## Licensing diff --git a/resources_servers/reasoning_gym/README.md b/resources_servers/reasoning_gym/README.md index 41ac8f2cc9..691109098c 100644 --- a/resources_servers/reasoning_gym/README.md +++ b/resources_servers/reasoning_gym/README.md @@ -82,15 +82,17 @@ policy_model_name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 ## Launch nemo gym servers ```bash -ng_run "+config_paths=[resources_servers/reasoning_gym/configs/reasoning_gym.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --resource-server reasoning_gym \ + --model-type vllm_model ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=reasoning_gym_simple_agent \ - +input_jsonl_fpath=resources_servers/reasoning_gym/data/example.jsonl \ - +output_jsonl_fpath=results/reasoning_gym_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent reasoning_gym_simple_agent \ + --input resources_servers/reasoning_gym/data/example.jsonl \ + --output results/reasoning_gym_rollouts.jsonl \ + --limit 5 ``` \ No newline at end of file diff --git a/resources_servers/simpleqa/README.md b/resources_servers/simpleqa/README.md index 0267583de9..c0dedfa747 100644 --- a/resources_servers/simpleqa/README.md +++ b/resources_servers/simpleqa/README.md @@ -39,20 +39,20 @@ the reasoning trace is split off before the response reaches this server. ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/simpleqa/configs/simpleqa.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --model-type vllm_model \ + --resource-server simpleqa \ +simpleqa.resources_servers.simpleqa.judge_model_server.name=policy_model ``` ## Collecting rollouts (5-example smoke test) ```bash -ng_collect_rollouts \ - +agent_name=simpleqa_simple_agent \ - +input_jsonl_fpath=resources_servers/simpleqa/data/example.jsonl \ - +output_jsonl_fpath=results/simpleqa_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent simpleqa_simple_agent \ + --input resources_servers/simpleqa/data/example.jsonl \ + --output results/simpleqa_rollouts.jsonl \ + --num-repeats 1 ``` ## Data Format diff --git a/resources_servers/single_step_tool_use_with_argument_comparison/README.md b/resources_servers/single_step_tool_use_with_argument_comparison/README.md index 7a489c15f4..adb3c6d98b 100644 --- a/resources_servers/single_step_tool_use_with_argument_comparison/README.md +++ b/resources_servers/single_step_tool_use_with_argument_comparison/README.md @@ -8,17 +8,17 @@ Data links: ? ## Running servers The following command can be used to run this resources server, along with the tool simulation agent and an OpenAI model: ```bash -config_paths="resources_servers/single_step_tool_use_with_argument_comparison/configs/single_step_tool_use_with_argument_comparison.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --resource-server single_step_tool_use_with_argument_comparison \ + --model-type openai_model ``` Then, rollouts can be collected using a command such as the following: ```bash -ng_collect_rollouts \ - +agent_name=single_step_tool_use_with_argument_comparison_agent \ - +input_jsonl_fpath=resources_servers/single_step_tool_use_with_argument_comparison/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/single_step_tool_use_with_argument_comparison/data/example_rollouts.jsonl +gym eval run --no-serve \ + --agent single_step_tool_use_with_argument_comparison_agent \ + --input resources_servers/single_step_tool_use_with_argument_comparison/data/example.jsonl \ + --output resources_servers/single_step_tool_use_with_argument_comparison/data/example_rollouts.jsonl ``` # Licensing information diff --git a/resources_servers/speed_bench/README.md b/resources_servers/speed_bench/README.md index 132b6783cb..7c727906e4 100644 --- a/resources_servers/speed_bench/README.md +++ b/resources_servers/speed_bench/README.md @@ -107,18 +107,18 @@ For an EAGLE3 / MTP setup with a paired draft model, see ```bash # Running servers — uses the demo local_vllm_model config above (drop in # your own model config to swap targets; just keep the speculative_config block). -config_paths="responses_api_models/local_vllm_model/configs/Qwen/Qwen3-30B-A3B-Instruct-2507-ngram-specdec.yaml,\ -resources_servers/speed_bench/configs/speed_bench.yaml,\ -responses_api_agents/speed_bench_agent/configs/speed_bench_agent.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --model-type local_vllm_model/Qwen/Qwen3-30B-A3B-Instruct-2507-ngram-specdec \ + --resource-server speed_bench \ + --config responses_api_agents/speed_bench_agent/configs/speed_bench_agent.yaml \ +policy_model=Qwen3-30B-A3B-Instruct-2507-ngram-specdec # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=speed_bench_simple_agent \ - +input_jsonl_fpath=resources_servers/speed_bench/data/example.jsonl \ - +output_jsonl_fpath=results/speed_bench_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent speed_bench_simple_agent \ + --input resources_servers/speed_bench/data/example.jsonl \ + --output results/speed_bench_rollouts.jsonl \ + --num-repeats 1 ``` If you'd rather use the lighter-weight `vllm_model` config (which expects @@ -129,7 +129,7 @@ when starting the upstream `vllm serve` process. ## Tests ```bash -ng_test +entrypoint=resources_servers/speed_bench +gym env test --resource-server speed_bench ``` The unit tests cover the Prometheus parser, the running-delta math, the diff --git a/resources_servers/spider2_lite/README.md b/resources_servers/spider2_lite/README.md index 1d8795dd7e..9d0b97defd 100644 --- a/resources_servers/spider2_lite/README.md +++ b/resources_servers/spider2_lite/README.md @@ -78,8 +78,10 @@ Each line must include either `gold_sql` (execute-and-compare) or `gold_result` Validate example data: ```bash -ng_prepare_data "+config_paths=[resources_servers/spider2_lite/configs/spider2_lite.yaml]" \ - +output_dirpath=/tmp/prepare +mode=example_validation +gym dataset collate \ + --resource-server spider2_lite \ + --output-dir /tmp/prepare \ + --mode example_validation ``` ## Known limitations diff --git a/resources_servers/structeval/README.md b/resources_servers/structeval/README.md index 6534ab7ab1..8d7715a181 100644 --- a/resources_servers/structeval/README.md +++ b/resources_servers/structeval/README.md @@ -28,19 +28,19 @@ reward = 0.2 * render_score + 0.8 * key_validation_score ### Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/structeval/configs/structeval_nonrenderable.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --model-type vllm_model \ + --resource-server structeval/structeval_nonrenderable ``` ### Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=structeval_nonrenderable_simple_agent \ - +input_jsonl_fpath=resources_servers/structeval/data/structeval_nonrenderable_train.jsonl \ - +output_jsonl_fpath=results/structeval_nonrenderable.jsonl \ - +resume_from_cache=True \ - +num_samples_in_parallel=256 +gym eval run --no-serve \ + --agent structeval_nonrenderable_simple_agent \ + --input resources_servers/structeval/data/structeval_nonrenderable_train.jsonl \ + --output results/structeval_nonrenderable.jsonl \ + --concurrency 256 \ + --resume ``` ## Data Preparation @@ -56,7 +56,7 @@ python resources_servers/structeval/misc/prepare_data.py \ ## Testing ```bash -ng_test +entrypoint=resources_servers/structeval +gym env test --resource-server structeval ``` ## Licensing diff --git a/resources_servers/structured_outputs/README.md b/resources_servers/structured_outputs/README.md index 50fa7d0c9e..9f2d449570 100644 --- a/resources_servers/structured_outputs/README.md +++ b/resources_servers/structured_outputs/README.md @@ -55,37 +55,37 @@ text-output and tool-call variants is in The following command runs the text-output JSON config with the simple agent and an OpenAI model: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -resources_servers/structured_outputs/configs/structured_outputs_json.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --resource-server structured_outputs/structured_outputs_json ``` Then, text-output rollouts can be collected using a command such as: ```bash -ng_collect_rollouts \ - +agent_name=structured_outputs_simple_agent \ - +input_jsonl_fpath=resources_servers/structured_outputs/data/structured_outputs_260309_nano_v3_sdg_json_yaml_xml_val.jsonl \ - +output_jsonl_fpath=results/example_structured_outputs_json.jsonl \ - +resume_from_cache=True \ - +num_samples_in_parallel=256 +gym eval run --no-serve \ + --agent structured_outputs_simple_agent \ + --input resources_servers/structured_outputs/data/structured_outputs_260309_nano_v3_sdg_json_yaml_xml_val.jsonl \ + --output results/example_structured_outputs_json.jsonl \ + --concurrency 256 \ + --resume ``` For v4 tool-call structured outputs, use `structured_outputs_v4.yaml` and the v4 simple agent. The config routes through a non-executing agent because the emitted function call is the final answer, not an action to execute: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/structured_outputs/configs/structured_outputs_v4.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --model-type vllm_model \ + --resource-server structured_outputs/structured_outputs_v4 ``` ```bash -ng_collect_rollouts \ - +agent_name=structured_outputs_v4_simple_agent \ - +input_jsonl_fpath=resources_servers/structured_outputs/data/structured_outputs_v4_tool_call.jsonl \ - +output_jsonl_fpath=results/example_structured_outputs_v4_tool_call.jsonl \ - +resume_from_cache=True \ - +num_samples_in_parallel=256 +gym eval run --no-serve \ + --agent structured_outputs_v4_simple_agent \ + --input resources_servers/structured_outputs/data/structured_outputs_v4_tool_call.jsonl \ + --output results/example_structured_outputs_v4_tool_call.jsonl \ + --concurrency 256 \ + --resume ``` You can see breakdown of results from the rollout file using the provided breakdown_metrics file. @@ -98,32 +98,33 @@ python resources_servers/structured_outputs/misc/breakdown_rollouts_metrics.py \ ### Version 1 [251027] (JSON only) You can prepare the data for training with: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/structured_outputs/configs/structured_outputs_json.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/structured_outputs \ - +mode=train_preparation +should_download=true +gym dataset collate \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --resource-server structured_outputs/structured_outputs_json \ + --output-dir data/structured_outputs \ + --mode train_preparation \ + --download ``` ### Version 2 [260310] (JSON, YAML, XML) ```bash # prepare -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/structured_outputs/configs/structured_outputs_json_yaml_xml_v1.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/structured_outputs/ \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resource-server structured_outputs/structured_outputs_json_yaml_xml_v1 \ + --output-dir data/structured_outputs/ \ + --mode train_preparation \ + --download ``` ### Version 3 [260409] (JSON, YAML, XML, TOML, CSV) ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/structured_outputs/configs/structured_outputs_v3.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/structured_outputs_v3/ \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resource-server structured_outputs/structured_outputs_v3 \ + --output-dir data/structured_outputs_v3/ \ + --mode train_preparation \ + --download ``` ### Version 4 [260424] (Tool-call structured outputs) @@ -139,12 +140,12 @@ The uploaded GitLab dataset is: Prepare the v4 training data with: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/structured_outputs/configs/structured_outputs_v4.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/structured_outputs_v4_tool_call/ \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resource-server structured_outputs/structured_outputs_v4 \ + --output-dir data/structured_outputs_v4_tool_call/ \ + --mode train_preparation \ + --download ``` The v4 config wires the train dataset to @@ -166,7 +167,7 @@ Generation details and CLI examples are in # Testing ``` -ng_test +entrypoint=resources_servers/structured_outputs +gym env test --resource-server structured_outputs ``` # Licensing information diff --git a/resources_servers/swerl_gen/README.md b/resources_servers/swerl_gen/README.md index 2a672bd632..6911dbe0f8 100644 --- a/resources_servers/swerl_gen/README.md +++ b/resources_servers/swerl_gen/README.md @@ -60,19 +60,18 @@ You need to have singularity installed. Image having singularity: `/lustre/fsw/portfolios/llmservice/users/asohrabizade/codegen/sqsh/nvidian+nemo+verl_v2_enroot0.8.5.sqsh` ```bash -config_paths="responses_api_agents/simple_agent/configs/simple_agent.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/swerl_gen/configs/swerl_gen.yaml" +gym env run \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type openai_model \ + --resource-server swerl_gen -ng_run "+config_paths=[$config_paths]" - -ng_collect_rollouts \ - +agent_name=swerl_gen_simple_agent \ - +input_jsonl_fpath=resources_servers/swerl_gen/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/swerl_gen/data/example_rollouts.jsonl \ - +num_repeats=2 \ - +num_samples_in_parallel=4 \ - +responses_create_params.max_output_tokens=4096 +gym eval run --no-serve \ + --agent swerl_gen_simple_agent \ + --input resources_servers/swerl_gen/data/example.jsonl \ + --output resources_servers/swerl_gen/data/example_rollouts.jsonl \ + --num-repeats 2 \ + --concurrency 4 \ + --max-output-tokens 4096 ``` Rollout example diff --git a/resources_servers/swerl_llm_judge/README.md b/resources_servers/swerl_llm_judge/README.md index 33149a1a4a..a5976891a1 100644 --- a/resources_servers/swerl_llm_judge/README.md +++ b/resources_servers/swerl_llm_judge/README.md @@ -55,20 +55,18 @@ Notes: **Standard format (with `grading_mode`):** ```bash -config_paths="responses_api_agents/simple_agent/configs/simple_agent.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/swerl_llm_judge/configs/swerl_llm_judge.yaml" - - -ng_run "+config_paths=[$config_paths]" - -ng_collect_rollouts \ - +agent_name=swerl_llm_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/swerl_llm_judge/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/swerl_llm_judge/data/example_rollouts.jsonl \ - +num_repeats=1 \ - +num_samples_in_parallel=5 \ - +responses_create_params.max_output_tokens=4096 +gym env run \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type openai_model \ + --resource-server swerl_llm_judge + +gym eval run --no-serve \ + --agent swerl_llm_judge_simple_agent \ + --input resources_servers/swerl_llm_judge/data/example.jsonl \ + --output resources_servers/swerl_llm_judge/data/example_rollouts.jsonl \ + --num-repeats 1 \ + --concurrency 5 \ + --max-output-tokens 4096 ``` diff --git a/resources_servers/tavily_search/README.md b/resources_servers/tavily_search/README.md index b4edfa0497..ca398a5235 100644 --- a/resources_servers/tavily_search/README.md +++ b/resources_servers/tavily_search/README.md @@ -27,29 +27,28 @@ tavily_search_resources_server: Commands to Run ``` -ng_download_dataset_from_gitlab \ - +dataset_name=tavily_search \ - +version=0.0.1 \ - +artifact_fpath=sft_samples_train.jsonl \ - +output_fpath=resources_servers/tavily_search/data/sft_samples/sft_samples_train.jsonl - -ng_download_dataset_from_gitlab \ - +dataset_name=tavily_search \ - +version=0.0.1 \ - +artifact_fpath=sft_samples_validation.jsonl \ - +output_fpath=resources_servers/tavily_search/data/sft_samples/sft_samples_validation.jsonl - -config_paths="resources_servers/tavily_search/configs/tavily_search_judge_vllm_model.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" - -ng_run "+config_paths=[${config_paths}]" +gym dataset download --storage gitlab \ + --name tavily_search \ + --revision 0.0.1 \ + --artifact sft_samples_train.jsonl \ + --output resources_servers/tavily_search/data/sft_samples/sft_samples_train.jsonl + +gym dataset download --storage gitlab \ + --name tavily_search \ + --revision 0.0.1 \ + --artifact sft_samples_validation.jsonl \ + --output resources_servers/tavily_search/data/sft_samples/sft_samples_validation.jsonl + +gym env run \ + --resource-server tavily_search/tavily_search_judge_vllm_model \ + --model-type vllm_model ``` ### Performance Metrics 100*16 samples: - Acc: 0.3212 -- Time in ng_collect: 44 mins +- Time in `gym eval run`: 44 mins # Licensing information diff --git a/resources_servers/terminus_judge/README.md b/resources_servers/terminus_judge/README.md index dd5dfbf3b5..4b97853ba1 100644 --- a/resources_servers/terminus_judge/README.md +++ b/resources_servers/terminus_judge/README.md @@ -32,19 +32,19 @@ For each verification request, the agent's JSON output is validated through mult The following command can be used to run this resources server, along with the simple agent and a policy model: ```bash -config_paths="resources_servers/terminus_judge/configs/terminus_judge.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --resource-server terminus_judge \ + --model-type openai_model \ +terminus_judge_resources_server.resources_servers.terminus_judge.judge_responses_create_params.max_output_tokens=512 ``` Then, rollouts can be collected using a command such as the following: ```bash -ng_collect_rollouts +agent_name=terminus_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/terminus_judge/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/terminus_judge/example_rollouts.jsonl +gym eval run --no-serve \ + --agent terminus_judge_simple_agent \ + --input resources_servers/terminus_judge/data/example.jsonl \ + --output resources_servers/terminus_judge/example_rollouts.jsonl ``` ## Expected Data Format diff --git a/resources_servers/terminus_judge/scripts/README.md b/resources_servers/terminus_judge/scripts/README.md index 99df28f8dd..63bb28a7e8 100644 --- a/resources_servers/terminus_judge/scripts/README.md +++ b/resources_servers/terminus_judge/scripts/README.md @@ -12,7 +12,7 @@ This pipeline uses the public `open-thoughts/OpenThoughts-Agent-v1-SFT` dataset - `datasets` - `openapi-schema-validator` 2. For smoke stage: - - `ng_run`, `ng_status`, `ng_collect_rollouts` on `PATH` + - `gym env run`, `gym env status`, `gym eval run --no-serve` on `PATH` - reachable policy/model endpoint 3. If HF cache is read-only: - `HF_HOME=/tmp/hf_home` diff --git a/resources_servers/text_to_sql/README.md b/resources_servers/text_to_sql/README.md index 741d423f82..45ab4c0373 100644 --- a/resources_servers/text_to_sql/README.md +++ b/resources_servers/text_to_sql/README.md @@ -68,19 +68,19 @@ Each data sample should include: ### Running Servers ```bash -config_paths="resources_servers/text_to_sql/configs/text_to_sql.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[${config_paths}]" \ +gym env run \ + --resource-server text_to_sql \ + --model-type openai_model \ +text_to_sql_resources_server.resources_servers.text_to_sql.judge_responses_create_params.max_output_tokens=512 ``` ### Collecting Rollouts ```bash -ng_collect_rollouts +agent_name=text_to_sql_simple_agent \ - +input_jsonl_fpath=resources_servers/text_to_sql/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/text_to_sql/data/example_rollouts.jsonl +gym eval run --no-serve \ + --agent text_to_sql_simple_agent \ + --input resources_servers/text_to_sql/data/example.jsonl \ + --output resources_servers/text_to_sql/data/example_rollouts.jsonl ``` ## Configuration Options diff --git a/resources_servers/ugphysics_judge/README.md b/resources_servers/ugphysics_judge/README.md index 095d9bd2a2..ef61b34397 100644 --- a/resources_servers/ugphysics_judge/README.md +++ b/resources_servers/ugphysics_judge/README.md @@ -58,17 +58,17 @@ swap it for any other `responses_api_models/*` config that exposes a ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/ugphysics_judge/configs/ugphysics_judge.yaml,\ -benchmarks/ugphysics/judge_gptoss20b.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server ugphysics_judge \ + --config benchmarks/ugphysics/judge_gptoss20b.yaml # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=ugphysics_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/ugphysics_judge/data/example.jsonl \ - +output_jsonl_fpath=results/ugphysics_judge_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent ugphysics_judge_simple_agent \ + --input resources_servers/ugphysics_judge/data/example.jsonl \ + --output results/ugphysics_judge_rollouts.jsonl \ + --num-repeats 1 ``` ## Metrics diff --git a/resources_servers/verifif/README.md b/resources_servers/verifif/README.md index 325451b6b5..db57187f8d 100644 --- a/resources_servers/verifif/README.md +++ b/resources_servers/verifif/README.md @@ -31,16 +31,18 @@ policy_model_name: gpt-5-2025-08-07 # or gpt-4.1-2025-04-14 ```bash cd /path/to/Gym source .venv/bin/activate -ng_run "+config_paths=[resources_servers/verifif/configs/verifif.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env run \ + --resource-server verifif \ + --model-type openai_model ``` ### 3. Run a test ```bash -ng_collect_rollouts \ - +agent_name=verifif_simple_agent \ - +input_jsonl_fpath=resources_servers/verifif/data/example.jsonl \ - +output_jsonl_fpath=results.jsonl +gym eval run --no-serve \ + --agent verifif_simple_agent \ + --input resources_servers/verifif/data/example.jsonl \ + --output results.jsonl ``` ## Architecture @@ -243,7 +245,7 @@ For high-throughput training: View server logs: ```bash -# Check terminal output from ng_run +# Check terminal output from gym env run # Or view Ray dashboard at http://127.0.0.1:8265 ``` diff --git a/resources_servers/vlm_eval_kit/README.md b/resources_servers/vlm_eval_kit/README.md index 32cdcccb0a..ee924cfc81 100644 --- a/resources_servers/vlm_eval_kit/README.md +++ b/resources_servers/vlm_eval_kit/README.md @@ -54,9 +54,9 @@ python run.py --verbose \ ### Prepare data First run the VLMEvalKit server to install dependencies. ```bash -config_paths="resources_servers/vlm_eval_kit/configs/vlm_eval_kit.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --resource-server vlm_eval_kit \ + --model-type openai_model ``` Then cd into this directory and activate the Python environment @@ -75,15 +75,14 @@ python prepare_data.py ```bash WANDB_PROJECT= EXPERIMENT_NAME=vlmevalkit/gpt-4o-mini-20240718 -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/vlm_eval_kit/configs/vlm_eval_kit.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ +gym eval run \ + --model-type openai_model \ + --resource-server vlm_eval_kit \ + --output results/$EXPERIMENT_NAME.jsonl \ + --split validation \ + --model gpt-4o-mini-2024-07-18 \ +wandb_project=$WANDB_PROJECT \ - +wandb_name=$EXPERIMENT_NAME \ - ++output_jsonl_fpath=results/$EXPERIMENT_NAME.jsonl \ - ++split=validation \ - ++policy_model_name=gpt-4o-mini-2024-07-18 + +wandb_name=$EXPERIMENT_NAME ``` # Licensing information diff --git a/resources_servers/wmt_translation/README.md b/resources_servers/wmt_translation/README.md index a649cbb04b..065116ceb2 100644 --- a/resources_servers/wmt_translation/README.md +++ b/resources_servers/wmt_translation/README.md @@ -70,17 +70,17 @@ For an end-to-end SLURM run with COMET enabled, see the ```bash # Running servers (BLEU-only locally; flip compute_comet=true on cluster) -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/wmt_translation/configs/wmt_translation.yaml" -ng_run "+config_paths=[$config_paths]" \ - "++wmt_translation.resources_servers.wmt_translation.compute_comet=false" +gym env run \ + --model-type vllm_model \ + --resource-server wmt_translation \ + ++wmt_translation.resources_servers.wmt_translation.compute_comet=false # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=wmt_translation_simple_agent \ - +input_jsonl_fpath=resources_servers/wmt_translation/data/example.jsonl \ - +output_jsonl_fpath=results/wmt_translation_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent wmt_translation_simple_agent \ + --input resources_servers/wmt_translation/data/example.jsonl \ + --output results/wmt_translation_rollouts.jsonl \ + --num-repeats 1 ``` For a fully reproducible end-to-end SLURM run that brings up vLLM with diff --git a/resources_servers/workplace_assistant/README.md b/resources_servers/workplace_assistant/README.md index c61e15945d..0bc16882fb 100644 --- a/resources_servers/workplace_assistant/README.md +++ b/resources_servers/workplace_assistant/README.md @@ -11,17 +11,18 @@ Commands - Spin up server: ``` -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/workplace_assistant/configs/workplace_assistant.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type openai_model \ + --resource-server workplace_assistant ``` Collect trajectories: ``` -ng_collect_rollouts +agent_name=workplace_assistant_simple_agent \ - +input_jsonl_fpath=resources_servers/workplace_assistant/data/train.jsonl \ - +output_jsonl_fpath=results/workplace_assistant_trajectory_collection.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent workplace_assistant_simple_agent \ + --input resources_servers/workplace_assistant/data/train.jsonl \ + --output results/workplace_assistant_trajectory_collection.jsonl \ + --limit 1 ``` ## Generating Additional Training Data diff --git a/resources_servers/xlam_fc/README.md b/resources_servers/xlam_fc/README.md index 06978e9718..5de238d123 100644 --- a/resources_servers/xlam_fc/README.md +++ b/resources_servers/xlam_fc/README.md @@ -9,17 +9,17 @@ python resources_servers/xlam_fc/generate_dataset.py ``` ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/xlam_fc/configs/xlam_fc.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run \ + --model-type vllm_model \ + --resource-server xlam_fc ``` ```bash -ng_collect_rollouts \ - +agent_name=xlam_fc_simple_agent \ - +input_jsonl_fpath=resources_servers/xlam_fc/data/train.jsonl \ - +output_jsonl_fpath=results/xlam_fc_trajectory_collection.jsonl \ - +limit=10 +gym eval run --no-serve \ + --agent xlam_fc_simple_agent \ + --input resources_servers/xlam_fc/data/train.jsonl \ + --output results/xlam_fc_trajectory_collection.jsonl \ + --limit 10 ``` ## Licensing diff --git a/resources_servers/xstest/README.md b/resources_servers/xstest/README.md index 11ef09bf23..8be3b56e70 100644 --- a/resources_servers/xstest/README.md +++ b/resources_servers/xstest/README.md @@ -83,7 +83,7 @@ Edge cases: Recommended generation parameters for benchmarking: ```bash -ng_collect_rollouts ... "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 32768}" +gym eval run --no-serve ... --temperature 1.0 --top-p 0.95 --max-output-tokens 32768 ``` Note: some models (e.g., Anthropic via Bedrock) do not allow `temperature` and `top_p` together. In that case, drop `top_p`. Use `temperature: 0.0` for deterministic/reproducible runs. @@ -100,16 +100,18 @@ contrast_privacy ### Example usage ```bash # For chat completions endpoints (vLLM, NIM, etc.): -ng_run "+config_paths=[resources_servers/xstest/configs/xstest.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --resource-server xstest \ + --model-type vllm_model # For OpenAI Responses API endpoints: -# ng_run "+config_paths=[resources_servers/xstest/configs/xstest.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +# gym env run --resource-server xstest --model-type openai_model -ng_collect_rollouts \ - +agent_name=xstest_simple_agent \ - +input_jsonl_fpath=resources_servers/xstest/data/example.jsonl \ - +output_jsonl_fpath=results/xstest_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent xstest_simple_agent \ + --input resources_servers/xstest/data/example.jsonl \ + --output results/xstest_rollouts.jsonl \ + --num-repeats 1 # Aggregate results python resources_servers/xstest/scripts/aggregate_results.py \ diff --git a/responses_api_agents/claude_code_agent/README.md b/responses_api_agents/claude_code_agent/README.md index fb6c65f5e2..dff1c0dcb4 100644 --- a/responses_api_agents/claude_code_agent/README.md +++ b/responses_api_agents/claude_code_agent/README.md @@ -29,17 +29,17 @@ anthropic_base_url: http://localhost:8000 No model server is needed for basic eval. To extend this agent to training, a model server should be developed that handles messages endpoint. For evals with the current version, just pass the resources server config, which includes the agent server config, as is the current standard in NeMo Gym: ```bash -ng_run "+config_paths=[resources_servers/reasoning_gym/configs/reasoning_gym_claude_code_agent.yaml]" +gym env run --resource-server reasoning_gym/reasoning_gym_claude_code_agent ``` ### Run the agent ```bash -ng_collect_rollouts \ - +agent_name=reasoning_gym_claude_code_agent \ - +input_jsonl_fpath=resources_servers/reasoning_gym/data/example.jsonl \ - +output_jsonl_fpath=claude_code_rollout.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent reasoning_gym_claude_code_agent \ + --input resources_servers/reasoning_gym/data/example.jsonl \ + --output claude_code_rollout.jsonl \ + --limit 1 ``` ## Description diff --git a/responses_api_agents/harbor_agent/README.md b/responses_api_agents/harbor_agent/README.md index 46e62cdcd0..29789759b9 100644 --- a/responses_api_agents/harbor_agent/README.md +++ b/responses_api_agents/harbor_agent/README.md @@ -200,9 +200,9 @@ export APPTAINER_DOCKER_PASSWORD= Then start NeMo Gym: ```bash -config_paths="responses_api_agents/harbor_agent/configs/harbor_agent.yaml,\ -responses_api_models/vllm_model/configs/vllm_model_for_training.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --config responses_api_agents/harbor_agent/configs/harbor_agent.yaml \ + --model-type vllm_model/vllm_model_for_training ``` ### 6) Test Harbor agent @@ -220,9 +220,10 @@ timestamped job directory containing per-trial outputs and a top-level ### 7) Collect rollouts ```bash -ng_collect_rollouts +agent_name=harbor_agent \ - +input_jsonl_fpath=responses_api_agents/harbor_agent/example/example_input.jsonl \ - +output_jsonl_fpath=responses_api_agents/harbor_agent/example/example_output.jsonl +gym eval run --no-serve \ + --agent harbor_agent \ + --input responses_api_agents/harbor_agent/example/example_input.jsonl \ + --output responses_api_agents/harbor_agent/example/example_output.jsonl ``` ### 8) View trajectories @@ -288,26 +289,29 @@ policy_model_name: Then follow the same Harbor-agent workflow with the Daytona config: ```bash -config_paths="responses_api_agents/harbor_agent/configs/harbor_agent_daytona.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env run \ + --config responses_api_agents/harbor_agent/configs/harbor_agent_daytona.yaml \ + --model-type vllm_model ``` Alternatively, pass those values as CLI overrides: ```bash -ng_run "+config_paths=[${config_paths}]" \ - +policy_base_url= \ - +policy_api_key= \ - +policy_model_name= +gym env run \ + --config responses_api_agents/harbor_agent/configs/harbor_agent_daytona.yaml \ + --model-type vllm_model \ + --model-url \ + --model-api-key \ + --model ``` For five Terminal-Bench rollout inputs, use the checked-in input file: ```bash -ng_collect_rollouts +agent_name=harbor_agent \ - +input_jsonl_fpath=responses_api_agents/harbor_agent/example/terminal_bench_daytona_input.jsonl \ - +output_jsonl_fpath=/tmp/harbor_daytona_terminal_bench_output.jsonl +gym eval run --no-serve \ + --agent harbor_agent \ + --input responses_api_agents/harbor_agent/example/terminal_bench_daytona_input.jsonl \ + --output /tmp/harbor_daytona_terminal_bench_output.jsonl ``` Inspect the rollout and Harbor job directory: diff --git a/responses_api_agents/hermes_agent/README.md b/responses_api_agents/hermes_agent/README.md index 7fad332aca..6ba99677ba 100644 --- a/responses_api_agents/hermes_agent/README.md +++ b/responses_api_agents/hermes_agent/README.md @@ -13,17 +13,19 @@ policy_model_name: gpt-4o ## Launch nemo gym servers ```bash -ng_run "+config_paths=[resources_servers/math_with_judge/configs/math_with_judge_hermes_agent.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env run \ + --resource-server math_with_judge/math_with_judge_hermes_agent \ + --model-type openai_model ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=math_with_judge_hermes_agent \ - +input_jsonl_fpath=resources_servers/math_with_judge/data/example.jsonl \ - +output_jsonl_fpath=hermes_agent_rollout.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent math_with_judge_hermes_agent \ + --input resources_servers/math_with_judge/data/example.jsonl \ + --output hermes_agent_rollout.jsonl \ + --limit 1 ``` 5 example math rollouts are at `responses_api_agents/hermes_agent/data/` with statistics: diff --git a/responses_api_agents/langgraph_agent/README.md b/responses_api_agents/langgraph_agent/README.md index 99585f5786..88b36d591b 100644 --- a/responses_api_agents/langgraph_agent/README.md +++ b/responses_api_agents/langgraph_agent/README.md @@ -9,13 +9,15 @@ Please note that agents such as parallel thinking which produce non-monotonicall ## Quick Start ```bash -ng_run "+config_paths=[resources_servers/reasoning_gym/configs/reflection_agent.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --resource-server reasoning_gym/reflection_agent \ + --model-type vllm_model ``` ```bash -ng_collect_rollouts \ - +agent_name=reasoning_gym_reflection_agent \ - +input_jsonl_fpath=resources_servers/reasoning_gym/data/example.jsonl \ - +output_jsonl_fpath=example_rollouts.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent reasoning_gym_reflection_agent \ + --input resources_servers/reasoning_gym/data/example.jsonl \ + --output example_rollouts.jsonl \ + --limit 1 ``` diff --git a/responses_api_agents/mini_swe_agent/README.md b/responses_api_agents/mini_swe_agent/README.md index 370bfb29a5..57d5fa1d0f 100644 --- a/responses_api_agents/mini_swe_agent/README.md +++ b/responses_api_agents/mini_swe_agent/README.md @@ -100,26 +100,28 @@ For how to download images and convert to .sif, you can refer to https://github. ```bash # Download swe-gym data -ng_download_dataset_from_gitlab \ - +dataset_name=mini_swe_agent \ - +version=0.0.1 \ - +artifact_fpath=train.jsonl \ - +output_fpath=data/train.jsonl +gym dataset download --storage gitlab \ + --name mini_swe_agent \ + --revision 0.0.1 \ + --artifact train.jsonl \ + --output data/train.jsonl # Start server -CONFIG_PATHS="resources_servers/mini_swe_agent/configs/mini_swe_agent.yaml,responses_api_models/openai_model/configs/openai_model.yaml" -ng_run +config_paths=[$CONFIG_PATHS] \ +gym env run \ + --resource-server mini_swe_agent \ + --model-type openai_model & \ '+mini_swe_simple_agent.responses_api_agents.mini_swe_agent.cache_dir_template=/path/to/images/xingyaoww_sweb.eval.x86_64.\{instance_id\}.sif' \ +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.run_golden=False \ +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.skip_if_exists=True \ +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.concurrency=16 \ +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.step_timeout=300 \ - +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.eval_timeout=900 & + +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.eval_timeout=900 # Collect rollouts -ng_collect_rollouts +agent_name=mini_swe_simple_agent \ - +input_jsonl_fpath=data/train.jsonl \ - +output_jsonl_fpath=results/mini_swe_agent_swe_gym.jsonl +gym eval run --no-serve \ + --agent mini_swe_simple_agent \ + --input data/train.jsonl \ + --output results/mini_swe_agent_swe_gym.jsonl ``` ### Training Setup and Results diff --git a/responses_api_agents/stirrup_agent/README.md b/responses_api_agents/stirrup_agent/README.md index 8ad50604ae..c07ddd7ba5 100644 --- a/responses_api_agents/stirrup_agent/README.md +++ b/responses_api_agents/stirrup_agent/README.md @@ -86,23 +86,22 @@ For each GDPVal task, the agent: The canonical entry point for GDPVal is the benchmark at [`benchmarks/gdpval/`](../../benchmarks/gdpval/README.md), which composes this agent with the GDPVal resources server and supports -`ng_prepare_benchmark` + `ng_e2e_collect_rollouts`: +`gym eval prepare` + `gym eval run`: ```bash # 1. Prepare the GDPVal benchmark JSONL. -ng_prepare_benchmark "+config_paths=[benchmarks/gdpval/config.yaml]" +gym eval prepare --benchmark gdpval # 2. Collect rollouts end-to-end (servers spin up automatically). -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -benchmarks/gdpval/config.yaml" JUDGE_API_KEY=... HF_TOKEN=... \ -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++split=benchmark \ - ++output_jsonl_fpath=results/gdpval_rubric.jsonl \ - ++policy_base_url=https://api.openai.com/v1 \ - ++policy_api_key=$OPENAI_API_KEY \ - ++policy_model_name=gpt-4.1-2025-04-14 +gym eval run \ + --model-type openai_model \ + --benchmark gdpval \ + --split benchmark \ + --output results/gdpval_rubric.jsonl \ + --model-url https://api.openai.com/v1 \ + --model-api-key $OPENAI_API_KEY \ + --model gpt-4.1-2025-04-14 ``` Each output line contains `responses_create_params`, the full `response`, a @@ -175,7 +174,7 @@ apptainer build gdpval.sif responses_api_agents/stirrup_agent/containers/gdpval. Then: ```yaml -# env.yaml or ng_run override +# env.yaml or gym env run override stirrup_agent: responses_api_agents: stirrup_agent: @@ -188,10 +187,11 @@ Pairwise comparison vs. a reference model is built into the GDPVal resources server (`resources_servers/gdpval`). Drive it from the benchmark config: ```bash -ng_e2e_collect_rollouts \ - "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/gdpval/config.yaml]" \ - ++split=benchmark \ - ++output_jsonl_fpath=results/gdpval_compare.jsonl \ +gym eval run \ + --model-type vllm_model \ + --benchmark gdpval \ + --split benchmark \ + --output results/gdpval_compare.jsonl \ ++gdpval_resources_server.resources_servers.gdpval.reward_mode=comparison \ ++gdpval_resources_server.resources_servers.gdpval.reference_deliverables_dir=output/gdpval/reference-model ``` diff --git a/responses_api_agents/swe_agents/README.md b/responses_api_agents/swe_agents/README.md index 8a8b7efc7b..b041564a94 100644 --- a/responses_api_agents/swe_agents/README.md +++ b/responses_api_agents/swe_agents/README.md @@ -251,15 +251,12 @@ policy_model_name: Qwen/Qwen3-Coder-30B-A3B-Instruct ### Step 2 — start the SWE-agents server ```bash -# OpenHands single-prompt -config_paths="responses_api_agents/swe_agents/configs/swebench_openhands.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" - -# Or full prompt × agent-class × tool-name diversity -config_paths="responses_api_agents/swe_agents/configs/swebench_multi_tools.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" - -ng_run "+config_paths=[$config_paths]" \ +# OpenHands single-prompt (swap to +# responses_api_agents/swe_agents/configs/swebench_multi_tools.yaml for full +# prompt × agent-class × tool-name diversity) +gym env run \ + --config responses_api_agents/swe_agents/configs/swebench_openhands.yaml \ + --model-type vllm_model \ +swe_agents.responses_api_agents.swe_agents.container_formatter=/lustre/xxx/images/swe-bench/swebench_sweb.eval.x86_64.\{instance_id\}.sif \ +swe_agents.responses_api_agents.swe_agents.model_server.name=vllm_model ``` @@ -284,14 +281,16 @@ python responses_api_agents/swe_agents/client.py ## Batch evaluation / data collection ```bash -ng_collect_rollouts +agent_name=swe_agents \ - +input_jsonl_fpath=swebench-verified-converted.jsonl \ - +output_jsonl_fpath=swebench-verified.openhands.qwen3-30b-coder.jsonl \ - +model=Qwen/Qwen3-Coder-30B-A3B-Instruct \ - +temperature=0.7 +top_p=0.8 +gym eval run --no-serve \ + --agent swe_agents \ + --input swebench-verified-converted.jsonl \ + --output swebench-verified.openhands.qwen3-30b-coder.jsonl \ + --model Qwen/Qwen3-Coder-30B-A3B-Instruct \ + --temperature 0.7 \ + --top-p 0.8 ``` -`ng_collect_rollouts` defaults to a concurrency of 100; tune to your hardware. View the results with: +`gym eval run` defaults to a concurrency of 100; tune to your hardware. View the results with: ```bash ng_viewer +jsonl_fpath=swebench-verified.openhands.qwen3-30b-coder.jsonl diff --git a/responses_api_agents/tau2/README.md b/responses_api_agents/tau2/README.md index 5a3d2a3a26..605dcc1ba1 100644 --- a/responses_api_agents/tau2/README.md +++ b/responses_api_agents/tau2/README.md @@ -1,8 +1,8 @@ # Description ```bash -config_paths="benchmarks/tau2/config.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_run "+config_paths=[${config_paths}]" \ +gym env run \ + --benchmark tau2 \ + --model-type openai_model \ ++nemo_gym_log_dir=results/tau2 \ '++gpt-5_2-2025-12-11.responses_api_models.openai_model.openai_api_key=${openai_api_key}' \ '++gpt-5_2-2025-12-11.responses_api_models.openai_model.extra_body._delete_key=max_output_tokens' diff --git a/responses_api_agents/verifiers_agent/README.md b/responses_api_agents/verifiers_agent/README.md index 14a8f4a541..017edc1684 100644 --- a/responses_api_agents/verifiers_agent/README.md +++ b/responses_api_agents/verifiers_agent/README.md @@ -21,14 +21,16 @@ policy_model_name: "Qwen/Qwen3-4B-Instruct-2507" ``` # start nemo gym servers -ng_run "+config_paths=[responses_api_agents/verifiers_agent/configs/acereason-math.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --config responses_api_agents/verifiers_agent/configs/acereason-math.yaml \ + --model-type vllm_model # generate a rollout -ng_collect_rollouts \ - +agent_name=verifiers_agent \ - +input_jsonl_fpath=responses_api_agents/verifiers_agent/data/acereason-math-example.jsonl \ - +output_jsonl_fpath=responses_api_agents/verifiers_agent/data/acereason-math-example-rollouts.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent verifiers_agent \ + --input responses_api_agents/verifiers_agent/data/acereason-math-example.jsonl \ + --output responses_api_agents/verifiers_agent/data/acereason-math-example-rollouts.jsonl \ + --limit 1 # view the rollout tail -n 1 responses_api_agents/verifiers_agent/data/acereason-math-example-rollouts.jsonl | jq | less @@ -98,21 +100,23 @@ deactivate source .venv/bin/activate # start nemo gym servers -ng_run "+config_paths=[responses_api_agents/verifiers_agent/configs/ascii-tree.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run \ + --config responses_api_agents/verifiers_agent/configs/ascii-tree.yaml \ + --model-type vllm_model # generate a rollout -ng_collect_rollouts \ - +agent_name=verifiers_agent \ - +input_jsonl_fpath=responses_api_agents/verifiers_agent/data/ascii-tree-example.jsonl \ - +output_jsonl_fpath=responses_api_agents/verifiers_agent/data/ascii-tree-example-rollouts.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent verifiers_agent \ + --input responses_api_agents/verifiers_agent/data/ascii-tree-example.jsonl \ + --output responses_api_agents/verifiers_agent/data/ascii-tree-example-rollouts.jsonl \ + --limit 1 ``` ## Integration notes The patch to include prompt and generation token ids for preventing retokenization error when training with NeMo RL has been upstreamed into verifiers' `NeMoRLChatCompletionsClient`. We currently track verifiers `main` (`git+https://github.com/PrimeIntellect-ai/verifiers.git@main`) for this support; once verifiers `0.1.13` is released, the requirements pin will move to that tagged version. -For installing new prime environments and generating datasets, use a separate venv (outside of Gym) to avoid dependency conflicts with the `exclude-dependencies` section of Gym `pyproject.toml` and the server's pinned verifiers version. After generating your dataset, deactivate the separate venv and return to the Gym venv for running servers. Make sure to restart NeMo Gym servers with `ng_run` after any environment changes to ensure the pinned version of verifiers is used. +For installing new prime environments and generating datasets, use a separate venv (outside of Gym) to avoid dependency conflicts with the `exclude-dependencies` section of Gym `pyproject.toml` and the server's pinned verifiers version. After generating your dataset, deactivate the separate venv and return to the Gym venv for running servers. Make sure to restart NeMo Gym servers with `gym env run` after any environment changes to ensure the pinned version of verifiers is used. # Licensing information Code: Apache 2.0 diff --git a/responses_api_models/azure_openai_model/README.md b/responses_api_models/azure_openai_model/README.md index 4d795640cd..43563a8e2d 100644 --- a/responses_api_models/azure_openai_model/README.md +++ b/responses_api_models/azure_openai_model/README.md @@ -15,27 +15,26 @@ policy_model_name: gpt-5-nano ### Running the server Set the API version. It usually looks something like "2024-10-21". ```bash -config_paths="responses_api_models/azure_openai_model/configs/azure_openai_model.yaml, \ -resources_servers/equivalence_llm_judge/configs/equivalence_llm_judge.yaml" - -ng_run "+config_paths=[${config_paths}]" \ +gym env run \ + --model-type azure_openai_model \ + --resource-server equivalence_llm_judge \ +policy_model.responses_api_models.azure_openai_model.default_query.api-version= ``` ### Collecting Rollouts ```bash -ng_collect_rollouts \ - +agent_name=equivalence_llm_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/equivalence_llm_judge/data/example.jsonl \ - +output_jsonl_fpath=results/example_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent equivalence_llm_judge_simple_agent \ + --input resources_servers/equivalence_llm_judge/data/example.jsonl \ + --output results/example_rollouts.jsonl \ + --limit 5 ``` ### Test cases ```bash -ng_test +entrypoint=responses_api_models/azure_openai_model +gym env test +entrypoint=responses_api_models/azure_openai_model ``` ## Licensing information diff --git a/responses_api_models/local_vllm_model/README.md b/responses_api_models/local_vllm_model/README.md index 6fbbbe6989..6e78803523 100644 --- a/responses_api_models/local_vllm_model/README.md +++ b/responses_api_models/local_vllm_model/README.md @@ -5,12 +5,12 @@ Run this on a single GPU node! Set tensor_parallel_size * data_parallel_size to the number of GPUs on your node. For this single node config, data_parallel_size_local is equal to data_parallel_size ```bash -config_paths="resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,\ -responses_api_models/local_vllm_model/configs/nano_v3_single_node.yaml" -ng_run "+config_paths=[${config_paths}]" \ +gym env run \ + --resource-server example_single_tool_call \ + --config responses_api_models/local_vllm_model/configs/nano_v3_single_node.yaml &> temp.log & \ ++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.tensor_parallel_size=4 \ ++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.data_parallel_size=2 \ - ++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.data_parallel_size_local=2 &> temp.log & + ++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.data_parallel_size_local=2 ``` View the logs diff --git a/responses_api_models/local_vllm_model_proxy/README.md b/responses_api_models/local_vllm_model_proxy/README.md index d4e57d6dd0..fa87d6a548 100644 --- a/responses_api_models/local_vllm_model_proxy/README.md +++ b/responses_api_models/local_vllm_model_proxy/README.md @@ -3,9 +3,9 @@ # End-to-end test with GPT OSS 20B reasoning high ```bash -config_paths="responses_api_models/local_vllm_model/configs/openai/gpt-oss-20b-reasoning-high.yaml,\ -responses_api_models/local_vllm_model_proxy/configs/local_vllm_model_proxy.yaml" -ng_run "+config_paths=[${config_paths}]" \ +gym env run \ + --model-type local_vllm_model/openai/gpt-oss-20b-reasoning-high \ + --model-type local_vllm_model_proxy \ ++policy_model_proxy.responses_api_models.local_vllm_model_proxy.model_server.name=gpt-oss-20b-reasoning-high \ ++policy_model_proxy.responses_api_models.local_vllm_model_proxy.extra_body.max_tokens=10 ``` From 8d720782cecadd35c5a99479a57873d9db3fc571 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Tue, 23 Jun 2026 10:08:53 +0200 Subject: [PATCH 23/40] fix: handle ng_ commands with no registered gym command Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/legacy.py | 12 +++++++++++- tests/unit_tests/test_cli_legacy.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/nemo_gym/cli/legacy.py b/nemo_gym/cli/legacy.py index 837f4b52ba..2c5c058b73 100644 --- a/nemo_gym/cli/legacy.py +++ b/nemo_gym/cli/legacy.py @@ -69,7 +69,17 @@ def main() -> None: dispatch("nemo_gym.cli.general:reinstall", sys.argv[1:]) return - tokens = LEGACY[key] + tokens = LEGACY.get(key) + if tokens is None: + # Reached only if a new `ng_*` / `nemo_gym_*` console script is wired to `legacy:main` + # without a matching `LEGACY` entry (a packaging bug, caught by tests), or a user invokes + # an alias that no longer exists. Fail loudly with a non-zero exit code. + print( + f"⚠ `{alias}` has no known `gym` equivalent. Run `gym --help` to see available commands.", + file=sys.stderr, + ) + sys.exit(1) + print( f"⚠ `{alias}` is deprecated and will be removed in a future release; use `gym {' '.join(tokens)}` instead.", file=sys.stderr, diff --git a/tests/unit_tests/test_cli_legacy.py b/tests/unit_tests/test_cli_legacy.py index cad72c0a61..4364d37bc8 100644 --- a/tests/unit_tests/test_cli_legacy.py +++ b/tests/unit_tests/test_cli_legacy.py @@ -53,3 +53,15 @@ def test_legacy_command_shows_deprecation(self, monkeypatch: MonkeyPatch, capsys legacy.main() assert "deprecated" in capsys.readouterr().err + + def test_unknown_alias_exits_nonzero(self, monkeypatch: MonkeyPatch, capsys) -> None: + # An alias with no LEGACY mapping (stale script or user typo) must fail loudly, not KeyError. + monkeypatch.setattr(legacy, "gym_main", lambda: None) + monkeypatch.setattr(legacy, "dispatch", lambda *a, **k: None) + monkeypatch.setattr(sys, "argv", ["ng_does_not_exist"]) + + with pytest.raises(SystemExit) as exc_info: + legacy.main() + + assert exc_info.value.code == 1 + assert "no known `gym` equivalent" in capsys.readouterr().err From 26be13d1a1ab1aab8e84f690db2d91a6d4b41115 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Tue, 23 Jun 2026 10:29:39 +0200 Subject: [PATCH 24/40] fix: add QUERY_KEY_NAME constant and make it a reserved key Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/eval.py | 3 ++- nemo_gym/global_config.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nemo_gym/cli/eval.py b/nemo_gym/cli/eval.py index 9a6ce87e09..d3066754ac 100644 --- a/nemo_gym/cli/eval.py +++ b/nemo_gym/cli/eval.py @@ -36,6 +36,7 @@ from nemo_gym.global_config import ( JSON_OUTPUT_KEY_NAME, POLICY_MODEL_KEY_NAME, + QUERY_KEY_NAME, ROLLOUT_INDEX_KEY_NAME, TASK_INDEX_KEY_NAME, GlobalConfigDictParser, @@ -134,7 +135,7 @@ def list_benchmarks() -> None: # `gym search ` reuses this command, narrowing the listing to fuzzy matches across the # benchmark name, agent name, resource server name, dataset names, and domain. - query = global_config_dict.get("query") + query = global_config_dict.get(QUERY_KEY_NAME) if query: benchmarks = { name: bench diff --git a/nemo_gym/global_config.py b/nemo_gym/global_config.py index 006808a48f..bf3c0f3145 100644 --- a/nemo_gym/global_config.py +++ b/nemo_gym/global_config.py @@ -72,6 +72,7 @@ NEMO_GYM_LOG_DIR_KEY_NAME = "nemo_gym_log_dir" VERBOSE_KEY_NAME = "verbose" JSON_OUTPUT_KEY_NAME = "json" +QUERY_KEY_NAME = "query" NEMO_GYM_RESERVED_TOP_LEVEL_KEYS = [ CONFIG_PATHS_KEY_NAME, ENTRYPOINT_KEY_NAME, @@ -96,6 +97,7 @@ NEMO_GYM_LOG_DIR_KEY_NAME, VERBOSE_KEY_NAME, JSON_OUTPUT_KEY_NAME, + QUERY_KEY_NAME, ] # Data keys From f83f70aba3896332c22fcad7d52c142cdd43a4c2 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Tue, 23 Jun 2026 11:08:47 +0200 Subject: [PATCH 25/40] fix: more descriptive error message when no benchmark configs found for benchmark preparation Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/eval.py | 14 ++++++++++++-- tests/unit_tests/test_benchmarks.py | 19 ++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/nemo_gym/cli/eval.py b/nemo_gym/cli/eval.py index d3066754ac..8d2d6259e6 100644 --- a/nemo_gym/cli/eval.py +++ b/nemo_gym/cli/eval.py @@ -228,11 +228,13 @@ def prepare_benchmark() -> None: prepare_benchmark_config = PrepareBenchmarkConfig.model_validate(global_config_dict) benchmarks_dict: Dict[str, BenchmarkConfig] = dict() + inspected_server_instances: List[str] = [] for server_instance_name in global_config_dict: server_config = global_config_dict[server_instance_name] if not isinstance(server_config, (dict, DictConfig)) or "responses_api_agents" not in server_config: continue + inspected_server_instances.append(server_instance_name) inner_server_config = get_first_server_config_dict(global_config_dict, server_instance_name) datasets: List[BenchmarkDatasetConfig] = [] @@ -246,7 +248,9 @@ def prepare_benchmark() -> None: continue assert len(datasets) == 1, ( - f"Expected 1 benchmark dataset for `{server_instance_name}`, but found {len(datasets)}!" + f"Expected exactly 1 benchmark dataset for server instance `{server_instance_name}`, " + f"but found {len(datasets)}: {[d.name for d in datasets]}. " + "A benchmark config must define a single benchmark dataset." ) dataset = datasets[0] @@ -260,7 +264,13 @@ def prepare_benchmark() -> None: ) assert benchmarks_dict, ( - 'No benchmark config found in config_paths. Pass a benchmark config, e.g.: "+config_paths=[benchmarks/aime24/config.yaml]"' + "No benchmark config found. " + + ( + f"Inspected server instances {inspected_server_instances}, but none declared a `benchmark` dataset." + if inspected_server_instances + else "No server instances with `responses_api_agents` were found in the resolved config." + ) + + " Pass a benchmark with `gym eval prepare --benchmark ` (e.g. `--benchmark aime24`)." ) # Validate all benchmarks before preparing any diff --git a/tests/unit_tests/test_benchmarks.py b/tests/unit_tests/test_benchmarks.py index 8ac1de1806..0858e3d88a 100644 --- a/tests/unit_tests/test_benchmarks.py +++ b/tests/unit_tests/test_benchmarks.py @@ -230,7 +230,24 @@ def test_no_benchmark_in_config_paths(self) -> None: ), patch("nemo_gym.cli.eval._load_benchmarks_from_config_paths", return_value={}), ): - with pytest.raises(AssertionError, match="No benchmark config found in config_paths"): + with pytest.raises(AssertionError, match="No benchmark config found"): + prepare_benchmark() + + def test_no_benchmark_dataset_reports_inspected_instances(self, tmp_path: Path) -> None: + # A server instance is present but declares no `benchmark` dataset; the error should name it + # so the user can see what was inspected. + config = { + "config_paths": ["benchmarks/dummy/config.yaml"], + "dummy_agent": { + "responses_api_agents": { + "simple_agent": { + "datasets": [{"name": "not_a_benchmark", "type": "train", "jsonl_fpath": str(tmp_path)}] + } + } + }, + } + with patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config(config)): + with pytest.raises(AssertionError, match=r"Inspected server instances \['dummy_agent'\]"): prepare_benchmark() def test_caching_sanity(self, tmp_path: Path) -> None: From a4c13b0ad27a737b3c2e089afd9abf3a39454c92 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Tue, 23 Jun 2026 13:05:44 +0200 Subject: [PATCH 26/40] fix: add backward compatibility for import used in RL Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/__init__.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/nemo_gym/cli/__init__.py b/nemo_gym/cli/__init__.py index 3159bfe656..9665e9afe6 100644 --- a/nemo_gym/cli/__init__.py +++ b/nemo_gym/cli/__init__.py @@ -12,3 +12,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from typing import Any + + +# NOTE: The cli module was re-structured but NeMo-RL relies on these two imports. +# They resolve lazily so importing this package (which happens on every `gym` invocation) +# doesn't eagerly pull in hydra, wandb and ray at startup. +_LEGACY_EXPORTS = { + "RunHelper": "nemo_gym.cli.env", + "GlobalConfigDictParserConfig": "nemo_gym.global_config", +} + + +def __getattr__(name: str) -> Any: + module_path = _LEGACY_EXPORTS.get(name) + if module_path is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + import importlib + + return getattr(importlib.import_module(module_path), name) From dd821a1d71eaaad2a3b16ddd6adb82c125b3f055 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 24 Jun 2026 08:59:42 +0200 Subject: [PATCH 27/40] fix: use year 2026 in SPDX header for new files Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/__init__.py | 2 +- nemo_gym/cli/dataset.py | 2 +- nemo_gym/cli/dev.py | 2 +- nemo_gym/cli/env.py | 2 +- nemo_gym/cli/eval.py | 2 +- nemo_gym/cli/general.py | 2 +- nemo_gym/cli/legacy.py | 2 +- nemo_gym/cli/main.py | 2 +- tests/unit_tests/test_cli_legacy.py | 2 +- tests/unit_tests/test_cli_main.py | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/nemo_gym/cli/__init__.py b/nemo_gym/cli/__init__.py index 9665e9afe6..be1e5395bb 100644 --- a/nemo_gym/cli/__init__.py +++ b/nemo_gym/cli/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/nemo_gym/cli/dataset.py b/nemo_gym/cli/dataset.py index eb3081ed4c..0984b99993 100644 --- a/nemo_gym/cli/dataset.py +++ b/nemo_gym/cli/dataset.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/nemo_gym/cli/dev.py b/nemo_gym/cli/dev.py index 2134d45d7a..a960f11b5c 100644 --- a/nemo_gym/cli/dev.py +++ b/nemo_gym/cli/dev.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/nemo_gym/cli/env.py b/nemo_gym/cli/env.py index 5a6b12b8ab..7554c0f625 100644 --- a/nemo_gym/cli/env.py +++ b/nemo_gym/cli/env.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/nemo_gym/cli/eval.py b/nemo_gym/cli/eval.py index 8d2d6259e6..f6ef237002 100644 --- a/nemo_gym/cli/eval.py +++ b/nemo_gym/cli/eval.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/nemo_gym/cli/general.py b/nemo_gym/cli/general.py index 3c577e8e88..6aca20da0e 100644 --- a/nemo_gym/cli/general.py +++ b/nemo_gym/cli/general.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/nemo_gym/cli/legacy.py b/nemo_gym/cli/legacy.py index 2c5c058b73..ba11c8eed9 100644 --- a/nemo_gym/cli/legacy.py +++ b/nemo_gym/cli/legacy.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index d935a335ae..baf191c9c6 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/tests/unit_tests/test_cli_legacy.py b/tests/unit_tests/test_cli_legacy.py index 4364d37bc8..1589b09a85 100644 --- a/tests/unit_tests/test_cli_legacy.py +++ b/tests/unit_tests/test_cli_legacy.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index bb57b44984..644ea060aa 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); From 7549fe7292c9248a9bd9a53fe66bd64e6498a48d Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 24 Jun 2026 09:26:03 +0200 Subject: [PATCH 28/40] fix: rename --resource-server -> --resources-server (typo) Signed-off-by: Marta Stepniewska-Dziubinska --- .claude/skills/add-benchmark/SKILL.md | 4 +- .../skills/nemo-gym-reward-profiling/SKILL.md | 2 +- .../skills/nemo-gym-reward-profiling/SKILL.md | 2 +- CLAUDE.md | 4 +- README.md | 2 +- benchmarks/labbench2_vlm/README.md | 2 +- .../pages/contribute/development-setup.mdx | 4 +- .../environments/new-environment.mdx | 4 +- .../pages/data/download-huggingface.mdx | 2 +- fern/versions/latest/pages/data/index.mdx | 4 +- .../latest/pages/data/prepare-validate.mdx | 6 +-- .../single-step-environment.mdx | 8 ++-- .../verification-patterns/llm-as-judge.mdx | 2 +- .../latest/pages/get-started/quickstart.mdx | 2 +- .../engineering-notes/system-design.mdx | 2 +- .../latest/pages/model-server/vllm.mdx | 4 +- .../latest/pages/reference/cli-commands.mdx | 38 +++++++++---------- fern/versions/latest/pages/reference/faq.mdx | 12 +++--- .../multi-environment-training.mdx | 8 ++-- .../training-tutorials/nemo-rl-grpo/setup.mdx | 2 +- .../latest/pages/training-tutorials/verl.mdx | 2 +- nemo_gym/cli/eval.py | 10 ++--- nemo_gym/cli/main.py | 30 +++++++-------- resources_servers/abstention/README.md | 2 +- resources_servers/arc_agi/README.md | 2 +- resources_servers/arena_judge/README.md | 2 +- resources_servers/asr_with_pc/README.md | 2 +- resources_servers/aviary/README.md | 6 +-- resources_servers/bigcodebench/README.md | 2 +- resources_servers/bird_sql/README.md | 2 +- resources_servers/blackjack/README.md | 2 +- .../browsecomp_advanced_harness/README.md | 2 +- resources_servers/calendar/README.md | 2 +- resources_servers/circle_click/README.md | 2 +- resources_servers/circle_count/README.md | 2 +- resources_servers/code_fim/README.md | 2 +- resources_servers/code_gen/README.md | 2 +- resources_servers/cvdp/README.md | 4 +- .../equivalence_llm_judge/README.md | 2 +- resources_servers/ether0/README.md | 2 +- resources_servers/evalplus/README.md | 2 +- .../example_multi_turn_gymnasium/README.md | 2 +- .../finance_sec_search/README.md | 6 +-- .../format_verification/README.md | 10 ++--- .../frontierscience_judge/README.md | 2 +- resources_servers/google_search/README.md | 2 +- resources_servers/gpqa_diamond/README.md | 4 +- resources_servers/graphwalks/README.md | 2 +- resources_servers/grl_sokoban/README.md | 2 +- resources_servers/grl_tetris/README.md | 2 +- resources_servers/hotpotqa_qa/README.md | 2 +- resources_servers/imo_gradingbench/README.md | 2 +- .../imo_proofbench_judge/README.md | 2 +- .../instruction_following/README.md | 4 +- resources_servers/inverse_if/README.md | 8 ++-- .../jailbreak_detection/README.md | 8 ++-- resources_servers/labbench2_vlm/README.md | 16 ++++---- resources_servers/longmt_eval/README.md | 2 +- .../math_advanced_calculations/README.md | 2 +- .../math_proof_judgement/README.md | 2 +- .../math_with_autograder/README.md | 4 +- resources_servers/math_with_code/README.md | 2 +- resources_servers/math_with_judge/README.md | 2 +- resources_servers/mcqa/README.md | 2 +- resources_servers/mrcr/README.md | 2 +- resources_servers/multichallenge/README.md | 8 ++-- resources_servers/newton_bench/README.md | 4 +- resources_servers/ns_tools/README.md | 6 +-- resources_servers/openenv/README.md | 8 ++-- resources_servers/physics_judge/README.md | 4 +- resources_servers/polymath/README.md | 2 +- resources_servers/rdkit_chemistry/README.md | 2 +- resources_servers/reasoning_gym/README.md | 2 +- resources_servers/simpleqa/README.md | 2 +- .../README.md | 2 +- resources_servers/speed_bench/README.md | 4 +- resources_servers/spider2_lite/README.md | 2 +- resources_servers/structeval/README.md | 4 +- .../structured_outputs/README.md | 14 +++---- resources_servers/swerl_gen/README.md | 2 +- resources_servers/swerl_llm_judge/README.md | 2 +- resources_servers/tavily_search/README.md | 2 +- resources_servers/terminus_judge/README.md | 2 +- resources_servers/text_to_sql/README.md | 2 +- resources_servers/ugphysics_judge/README.md | 2 +- resources_servers/verifif/README.md | 2 +- resources_servers/vlm_eval_kit/README.md | 4 +- resources_servers/wmt_translation/README.md | 2 +- .../workplace_assistant/README.md | 2 +- resources_servers/xlam_fc/README.md | 2 +- resources_servers/xstest/README.md | 4 +- .../claude_code_agent/README.md | 2 +- responses_api_agents/hermes_agent/README.md | 2 +- .../langgraph_agent/README.md | 2 +- responses_api_agents/mini_swe_agent/README.md | 2 +- .../azure_openai_model/README.md | 2 +- .../local_vllm_model/README.md | 2 +- tests/unit_tests/test_benchmarks.py | 4 +- tests/unit_tests/test_cli_main.py | 30 +++++++-------- 99 files changed, 214 insertions(+), 214 deletions(-) diff --git a/.claude/skills/add-benchmark/SKILL.md b/.claude/skills/add-benchmark/SKILL.md index 3ed1460d87..8b2c027831 100644 --- a/.claude/skills/add-benchmark/SKILL.md +++ b/.claude/skills/add-benchmark/SKILL.md @@ -35,7 +35,7 @@ Before starting, determine which type of benchmark you're adding: Run `gym env init` to generate the directory structure: ```bash -gym env init --resource-server my_benchmark +gym env init --resources-server my_benchmark ``` This creates: @@ -166,7 +166,7 @@ Both fields must coexist: `jsonl_fpath` is the local download destination, `gitl ```bash # Run server tests (creates isolated .venv, slow on first run) -gym env test --resource-server my_benchmark +gym env test --resources-server my_benchmark # Run core library tests to check nothing broke pytest tests/unit_tests/ -x diff --git a/.claude/skills/nemo-gym-reward-profiling/SKILL.md b/.claude/skills/nemo-gym-reward-profiling/SKILL.md index ab7067698d..20b42c7dab 100644 --- a/.claude/skills/nemo-gym-reward-profiling/SKILL.md +++ b/.claude/skills/nemo-gym-reward-profiling/SKILL.md @@ -14,7 +14,7 @@ description: >- Use this skill when the user wants to run, understand, or lightly modify Nemo Gym reward profiling. Keep the answer oriented around the normal workflow: -`gym env run` starts model/resource servers, `gym eval run --no-serve` writes rollout artifacts, and `gym eval profile` generates profiling output from those artifacts. +`gym env run` starts model/resources servers, `gym eval run --no-serve` writes rollout artifacts, and `gym eval profile` generates profiling output from those artifacts. If the user is primarily debugging a failed job or stack trace, use the `nemo-gym-debugging` skill first. diff --git a/.codex/skills/nemo-gym-reward-profiling/SKILL.md b/.codex/skills/nemo-gym-reward-profiling/SKILL.md index ab7067698d..20b42c7dab 100644 --- a/.codex/skills/nemo-gym-reward-profiling/SKILL.md +++ b/.codex/skills/nemo-gym-reward-profiling/SKILL.md @@ -14,7 +14,7 @@ description: >- Use this skill when the user wants to run, understand, or lightly modify Nemo Gym reward profiling. Keep the answer oriented around the normal workflow: -`gym env run` starts model/resource servers, `gym eval run --no-serve` writes rollout artifacts, and `gym eval profile` generates profiling output from those artifacts. +`gym env run` starts model/resources servers, `gym eval run --no-serve` writes rollout artifacts, and `gym eval profile` generates profiling output from those artifacts. If the user is primarily debugging a failed job or stack trace, use the `nemo-gym-debugging` skill first. diff --git a/CLAUDE.md b/CLAUDE.md index e7df93a3a7..39d3edbfd6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,12 +83,12 @@ pre-commit install # Run servers gym env run \ - --resource-server example_single_tool_call \ + --resources-server example_single_tool_call \ --model-type vllm_model # Run tests for a specific server (creates .venv per server, installs deps, runs pytest) # First run is slow. Use skip_venv_if_present config or place a .venv to skip venv creation. -gym env test --resource-server example_single_tool_call +gym env test --resources-server example_single_tool_call # Run all server tests gym env test diff --git a/README.md b/README.md index b5a6653869..22c1de02bb 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ NeMo Gym uses local servers to coordinate your model, agent, and task verificati ```bash gym env run \ - --resource-server mcqa \ + --resources-server mcqa \ --model-type openai_model ``` diff --git a/benchmarks/labbench2_vlm/README.md b/benchmarks/labbench2_vlm/README.md index 68ceb7fb31..a432ba413c 100644 --- a/benchmarks/labbench2_vlm/README.md +++ b/benchmarks/labbench2_vlm/README.md @@ -83,7 +83,7 @@ After changing `example.jsonl`, regenerate its static validation metrics: ```bash .venv/bin/gym dataset collate \ - --resource-server labbench2_vlm \ + --resources-server labbench2_vlm \ --config resources_servers/labbench2_vlm/configs/judge_model_openai.yaml \ --config responses_api_models/openai_model/configs/openai_model.yaml \ --mode example_validation \ diff --git a/fern/versions/latest/pages/contribute/development-setup.mdx b/fern/versions/latest/pages/contribute/development-setup.mdx index 9ca07cf367..6fbfad3f75 100644 --- a/fern/versions/latest/pages/contribute/development-setup.mdx +++ b/fern/versions/latest/pages/contribute/development-setup.mdx @@ -27,7 +27,7 @@ pre-commit install ```bash gym dev test # Run all NeMo Gym core tests gym env test # Run all server tests -gym env test --resource-server example_single_tool_call # Test a single resources server +gym env test --resources-server example_single_tool_call # Test a single resources server ``` **View Test Coverage**: @@ -55,7 +55,7 @@ All contributions must pass these automated checks: **Test Requirements**: - At least one test per server you contribute -- Tests must run using `gym env test --resource-server your_server` +- Tests must run using `gym env test --resources-server your_server` - Use pytest for async testing patterns ### Build Docs CI Failures diff --git a/fern/versions/latest/pages/contribute/environments/new-environment.mdx b/fern/versions/latest/pages/contribute/environments/new-environment.mdx index 728f700dad..92470cdcf5 100644 --- a/fern/versions/latest/pages/contribute/environments/new-environment.mdx +++ b/fern/versions/latest/pages/contribute/environments/new-environment.mdx @@ -63,7 +63,7 @@ Prepare the dataset for your environment: Build your resources server: -- Run `gym env init --resource-server my_server` to scaffold the new resources server +- Run `gym env init --resources-server my_server` to scaffold the new resources server - Follow the [Single Step Environment](/environment-tutorials/single-step-environment) guide to implement your specific logic - Implement verification logic for your tasks by defining the `verify()` function - Set the `domain` field in your resources server configuration (see `Domain`). @@ -80,7 +80,7 @@ Write and run tests for your resources server: Verify basic functionality and generate example rollouts: -- Document the command used to start your server, for example, `gym env run --resource-server my_server` +- Document the command used to start your server, for example, `gym env run --resources-server my_server` - Generate rollouts and save 5 example outputs to `data/example_rollouts.jsonl` to demonstrate correct reward signals ### 5. Reward Profiling diff --git a/fern/versions/latest/pages/data/download-huggingface.mdx b/fern/versions/latest/pages/data/download-huggingface.mdx index d1f237390f..bc6f3337e4 100644 --- a/fern/versions/latest/pages/data/download-huggingface.mdx +++ b/fern/versions/latest/pages/data/download-huggingface.mdx @@ -246,7 +246,7 @@ Run with download enabled: ```bash gym dataset collate \ - --resource-server code_gen \ + --resources-server code_gen \ --output-dir ./data/prepared \ --mode train_preparation \ --download \ diff --git a/fern/versions/latest/pages/data/index.mdx b/fern/versions/latest/pages/data/index.mdx index c8dc02d465..0bd227a052 100644 --- a/fern/versions/latest/pages/data/index.mdx +++ b/fern/versions/latest/pages/data/index.mdx @@ -77,7 +77,7 @@ Run this command from the repository root: ```bash gym dataset collate \ --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ - --resource-server example_multi_step \ + --resources-server example_multi_step \ --output-dir data/test \ --mode example_validation ``` @@ -142,7 +142,7 @@ To prepare training data with auto-download: ```bash gym dataset collate \ --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ - --resource-server workplace_assistant \ + --resources-server workplace_assistant \ --output-dir data/workplace_assistant \ --mode train_preparation \ --download diff --git a/fern/versions/latest/pages/data/prepare-validate.mdx b/fern/versions/latest/pages/data/prepare-validate.mdx index 503a69344b..e32bbad1f7 100644 --- a/fern/versions/latest/pages/data/prepare-validate.mdx +++ b/fern/versions/latest/pages/data/prepare-validate.mdx @@ -31,7 +31,7 @@ From the repository root: ```bash gym dataset collate \ - --resource-server example_multi_step \ + --resources-server example_multi_step \ --config responses_api_models/openai_model/configs/openai_model.yaml \ --output-dir data/test \ --mode example_validation @@ -237,7 +237,7 @@ This validates your data and adds the `agent_ref` field to each row, routing sam ```bash gym dataset collate \ - --resource-server example_multi_step \ + --resources-server example_multi_step \ --config responses_api_models/vllm_model/configs/vllm_model.yaml \ --output-dir data/example_multi_step \ --mode example_validation @@ -247,7 +247,7 @@ gym dataset collate \ ```bash gym dataset collate \ - --resource-server workplace_assistant \ + --resources-server workplace_assistant \ --config responses_api_models/vllm_model/configs/vllm_model.yaml \ --output-dir data/workplace_assistant \ --mode train_preparation \ diff --git a/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx b/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx index f635f028a5..4f123edf1c 100644 --- a/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx +++ b/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx @@ -53,7 +53,7 @@ In most cases, the **Resources Server** is where your changes go: define your to Resource servers live in the `resources_servers/` directory. Scaffold a weather server that provides weather information to models: ```bash -gym env init --resource-server my_weather_tool +gym env init --resources-server my_weather_tool ``` This generates the following structure along with a paired simple agent configuration: @@ -393,7 +393,7 @@ async def test_verify_without_tool_call(server): Run the tests: ```bash -gym env test --resource-server my_weather_tool +gym env test --resources-server my_weather_tool ``` For detailed test output: @@ -415,7 +415,7 @@ Start the servers: ```bash gym env run \ --model-type openai_model \ - --resource-server my_weather_tool + --resources-server my_weather_tool ``` `gym env run` reads the config files and starts all three components from the architecture diagram: @@ -545,7 +545,7 @@ We'd love to see your contributions! Please make sure your PR includes accurate You've learned how to: -- Initialize a resource server with `gym env init` +- Initialize a resources server with `gym env init` - Prepare task data in JSONL format - Implement tool endpoints and verification logic - Configure the required `domain` field and wire components together diff --git a/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx b/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx index 77417f09b2..bec06225fb 100644 --- a/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx +++ b/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx @@ -201,7 +201,7 @@ Start the servers: ```bash gym env run \ - --resource-server over_refusal_detection \ + --resources-server over_refusal_detection \ --model-type openai_model ``` diff --git a/fern/versions/latest/pages/get-started/quickstart.mdx b/fern/versions/latest/pages/get-started/quickstart.mdx index 7eec1d9929..bad7ef327a 100644 --- a/fern/versions/latest/pages/get-started/quickstart.mdx +++ b/fern/versions/latest/pages/get-started/quickstart.mdx @@ -32,7 +32,7 @@ NeMo Gym uses local servers to coordinate your model, agent, and task verificati ```bash gym env run \ - --resource-server mcqa \ + --resources-server mcqa \ --model-type openai_model ``` diff --git a/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx b/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx index 017101aa6f..55eacd2ac9 100644 --- a/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx +++ b/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx @@ -22,7 +22,7 @@ The `gym env run` command uses Hydra to parse command-line arguments. Users spec ```bash gym env run \ - --resource-server math \ + --resources-server math \ --model-type openai_model ``` diff --git a/fern/versions/latest/pages/model-server/vllm.mdx b/fern/versions/latest/pages/model-server/vllm.mdx index 9f6485c6f8..5b3cddd591 100644 --- a/fern/versions/latest/pages/model-server/vllm.mdx +++ b/fern/versions/latest/pages/model-server/vllm.mdx @@ -12,7 +12,7 @@ VLLMModel provides a Responses API to Chat Completions mapping middleware layer **To use VLLMModel, just change the `responses_api_models/openai_model/configs/openai_model.yaml` in your config paths to `responses_api_models/vllm_model/configs/vllm_model.yaml`!** ```bash gym env run \ - --resource-server example_multi_step \ + --resources-server example_multi_step \ --model-type vllm_model ``` @@ -117,7 +117,7 @@ vllm serve \ In a second terminal on the same GPU node that was used to spin up the vLLM server, enter the NeMo Gym Python environment, and start the NeMo Gym servers. ```bash gym env run \ - --resource-server example_multi_step \ + --resources-server example_multi_step \ --model-type vllm_model \ --model-url http://0.0.0.0:10240/v1 \ --model Qwen/Qwen3-4B-Thinking-2507 \ diff --git a/fern/versions/latest/pages/reference/cli-commands.mdx b/fern/versions/latest/pages/reference/cli-commands.mdx index 61851029fd..1c702245bd 100644 --- a/fern/versions/latest/pages/reference/cli-commands.mdx +++ b/fern/versions/latest/pages/reference/cli-commands.mdx @@ -35,7 +35,7 @@ gym dataset collate # validate and collate a dataset gym env init # scaffold a new resources server gym env resolve # resolve and print the final merged config gym env packages # list packages in a server's virtual environment -gym env test # test resource server(s); all of them if none is given +gym env test # test resources server(s); all of them if none is given gym env run # start the servers gym env status # show running servers @@ -57,7 +57,7 @@ These options are shared across many commands. | --- | --- | | `--config PATH` | Load a Gym config YAML. Repeatable. Maps to `+config_paths=[...]`. | | `--benchmark NAME` | Select a registered benchmark by name instead of a config path. | -| `--resource-server NAME` | Select a registered resources server by name. | +| `--resources-server NAME` | Select a registered resources server by name. | | `--model-type NAME` | Select a registered model server type by name (such as `openai_model` or `vllm_model`). | | `--search-dir DIR` | Extra root directory to search for named components. Repeatable. Lets you register your own benchmarks, resources servers, and models. | | `--json` | Emit machine-readable JSON instead of human-readable output (reporting commands only). | @@ -65,7 +65,7 @@ These options are shared across many commands. | `-h`, `--help` | Show help for any group or command. | -The `--benchmark`, `--resource-server`, and `--model-type` selectors resolve a component name to its config file for you, so you do not need to know the project's directory layout. If you mistype a name, the CLI suggests the closest match. To point at a config file directly, use `--config ` instead. +The `--benchmark`, `--resources-server`, and `--model-type` selectors resolve a component name to its config file for you, so you do not need to know the project's directory layout. If you mistype a name, the CLI suggests the closest match. To point at a config file directly, use `--config ` instead. ### Hydra overrides (escape hatch) @@ -276,7 +276,7 @@ Validate and collate a dataset, generating metrics and statistics. Replaces `ng_ | Option | Description | | --- | --- | | `--config PATH` | Config file to load. Repeatable. | -| `--resource-server NAME` | Load the named resources server config. | +| `--resources-server NAME` | Load the named resources server config. | | `--search-dir DIR` | Extra root directory to search for named components. Repeatable. | | `--mode {train_preparation,example_validation}` | Use `train_preparation` to prepare train/validation datasets, or `example_validation` to validate example data for PR submission. | | `--output-dir` | Output directory for the prepared data. | @@ -284,7 +284,7 @@ Validate and collate a dataset, generating metrics and statistics. Replaces `ng_ ```bash gym dataset collate \ - --resource-server example_multi_step \ + --resources-server example_multi_step \ --config responses_api_models/openai_model/configs/openai_model.yaml \ --output-dir data/example_multi_step \ --mode example_validation @@ -302,10 +302,10 @@ Scaffold a new resources server with template files (config, app, tests, README, | Option | Description | | --- | --- | -| `--resource-server NAME` | Name of the resources server to create. | +| `--resources-server NAME` | Name of the resources server to create. | ```bash -gym env init --resource-server my_server +gym env init --resources-server my_server ``` ### `gym env resolve` @@ -326,16 +326,16 @@ Each server has its own isolated virtual environment. List the packages installe | Option | Description | | --- | --- | -| `--resource-server NAME` | Name of the resources server. | +| `--resources-server NAME` | Name of the resources server. | | `--outdated` | List only outdated packages. | | `--json` | Output the package list as JSON. | ```bash -gym env packages --resource-server example_single_tool_call +gym env packages --resources-server example_single_tool_call # Check for outdated packages gym env packages \ - --resource-server example_single_tool_call \ + --resources-server example_single_tool_call \ --outdated ``` @@ -345,11 +345,11 @@ Test resource server(s) by running their pytest suite. If no resources server is | Option | Description | | --- | --- | -| `--resource-server NAME` | Resources server to test. Omit to test all servers. | +| `--resources-server NAME` | Resources server to test. Omit to test all servers. | ```bash # Test a single server -gym env test --resource-server example_single_tool_call +gym env test --resources-server example_single_tool_call # Test all servers gym env test @@ -363,7 +363,7 @@ Start the NeMo Gym servers (agents, models, resources) defined by the provided c | --- | --- | | `--config PATH` | Config file to load. Repeatable. | | `--benchmark NAME` | Load the named benchmark config (start its servers). | -| `--resource-server NAME` | Load the named resources server config. | +| `--resources-server NAME` | Load the named resources server config. | | `--model-type NAME` | Load the named model server type config. | | `--search-dir DIR` | Extra root directory to search for named components. Repeatable. | | `--model`, `-m` | Served model identifier. See [Selecting the model server](#selecting-the-model-server). | @@ -372,7 +372,7 @@ Start the NeMo Gym servers (agents, models, resources) defined by the provided c ```bash gym env run \ - --resource-server example_single_tool_call \ + --resources-server example_single_tool_call \ --model-type openai_model # Start a benchmark's servers @@ -437,7 +437,7 @@ Collate data, start the servers, and collect rollouts. This is the main evaluati | --- | --- | | `--config PATH` | Config file to load. Repeatable. | | `--benchmark NAME` | Load the named benchmark config. | -| `--resource-server NAME` | Load the named resources server config. | +| `--resources-server NAME` | Load the named resources server config. | | `--model-type NAME` | Load the named model server type config. | | `--search-dir DIR` | Extra root directory to search for named components. Repeatable. | | `--no-serve` | Collect against already-running servers instead of starting them. | @@ -468,7 +468,7 @@ gym eval run --benchmark aime24 \ # Against an already-running server, with a remote vLLM endpoint gym eval run --no-serve \ --model-type openai_model \ - --resource-server math_with_judge \ + --resources-server math_with_judge \ --output results/test_001.jsonl \ --split validation \ --model openai/gpt-oss-120b \ @@ -537,7 +537,7 @@ The legacy `ng_*` and `nemo_gym_*` console scripts are deprecated but still func | `ng_pip_list` | `gym env packages` | | `ng_init_resources_server` | `gym env init` | | `ng_test` | `gym env test` | -| `ng_test_all` | `gym env test` (no `--resource-server`) | +| `ng_test_all` | `gym env test` (no `--resources-server`) | | `ng_prepare_benchmark` | `gym eval prepare` | | `ng_e2e_collect_rollouts` | `gym eval run` | | `ng_collect_rollouts` | `gym eval run --no-serve` | @@ -586,7 +586,7 @@ The most common Hydra overrides now have dedicated flags. Prefer flags; fall bac ng_run "+config_paths=[resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" # After: gym env run \ - --resource-server example_single_tool_call \ + --resources-server example_single_tool_call \ --model-type openai_model # End-to-end rollout collection @@ -598,7 +598,7 @@ ng_e2e_collect_rollouts "+config_paths=[${config_paths}]" \ # After: gym eval run \ --model-type openai_model \ - --resource-server math_with_judge \ + --resources-server math_with_judge \ --output results/aime24.jsonl \ --split validation diff --git a/fern/versions/latest/pages/reference/faq.mdx b/fern/versions/latest/pages/reference/faq.mdx index b4e8f6538e..1eac05b8b2 100644 --- a/fern/versions/latest/pages/reference/faq.mdx +++ b/fern/versions/latest/pages/reference/faq.mdx @@ -145,7 +145,7 @@ gym dataset download \ Use `artifact_fpath` when the Hugging Face repo contains raw/arbitrary JSONL files rather than structured dataset splits. You cannot specify both `split` and `artifact_fpath`. # How To: Prepare and validate data for PR submission or RL training -When you use `gym env init --resource-server example_multi_step` to initialize a resources server, you will get a config.yaml that looks like the below code block. The dataset information for training, validation, and example will be inside the scope of your agent config (e.g. under simple_agent) and is a list of dataset objects. +When you use `gym env init --resources-server example_multi_step` to initialize a resources server, you will get a config.yaml that looks like the below code block. The dataset information for training, validation, and example will be inside the scope of your agent config (e.g. under simple_agent) and is a list of dataset objects. ```yaml example_multi_step_resources_server: @@ -223,7 +223,7 @@ Each config.yaml in the resources server requires at least one agent with one ex For every PR that contributes data, we require common dataset statistics and sanity checks on the data itself. This process is also helpful to catch any simple issues before you ever train with NeMo RL. NeMo Gym provides a helper command gym dataset collate to do so. ```bash gym dataset collate \ - --resource-server example_multi_step \ + --resources-server example_multi_step \ --config responses_api_models/openai_model/configs/openai_model.yaml \ --output-dir data/example_multi_step \ --mode example_validation @@ -232,7 +232,7 @@ gym dataset collate \ To download missing datasets automatically, add --download. By default, datasets are downloaded from Hugging Face: ```bash gym dataset collate \ - --resource-server example_multi_step \ + --resources-server example_multi_step \ --config responses_api_models/openai_model/configs/openai_model.yaml \ --output-dir data/example_multi_step \ --mode train_preparation \ @@ -243,7 +243,7 @@ For NVIDIA internal users, you can download from GitLab instead: ```bash gym dataset collate \ - --resource-server example_multi_step \ + --resources-server example_multi_step \ --config responses_api_models/openai_model/configs/openai_model.yaml \ --output-dir data/example_multi_step \ --mode train_preparation \ @@ -254,7 +254,7 @@ gym dataset collate \ Run NeMo Gym servers the exact same way with the same configs! ```bash gym env run \ - --resource-server example_multi_step \ + --resources-server example_multi_step \ --model-type openai_model ``` @@ -278,7 +278,7 @@ The `gym dataset collate` command will: The `gym dataset collate` command has 2 modes, one for actual train and validation set preparation, and one for example validation intended to sanity check your data format. You would typically run `--mode example_validation` when first contributing a resources server, and then run with `--mode train_preparation` when you actually go to train. ```bash gym dataset collate \ - --resource-server example_multi_step \ + --resources-server example_multi_step \ --config responses_api_models/openai_model/configs/openai_model.yaml \ --output-dir data/example_multi_step \ --mode example_validation diff --git a/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx b/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx index 5a114aaa50..5571bb6c68 100644 --- a/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx +++ b/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx @@ -15,22 +15,22 @@ For example_single_tool_call: ```bash gym env run \ --model-type openai_model \ - --resource-server example_single_tool_call + --resources-server example_single_tool_call ``` For example_multi_step: ```bash gym env run \ --model-type openai_model \ - --resource-server example_multi_step + --resources-server example_multi_step ``` To use both environments, add the YAML configs together as follows: ```bash gym env run \ --model-type openai_model \ - --resource-server example_single_tool_call \ - --resource-server example_multi_step + --resources-server example_single_tool_call \ + --resources-server example_multi_step ``` ## Dataset Preparation diff --git a/fern/versions/latest/pages/training-tutorials/nemo-rl-grpo/setup.mdx b/fern/versions/latest/pages/training-tutorials/nemo-rl-grpo/setup.mdx index 3f66e05efd..a3cc2cb6aa 100644 --- a/fern/versions/latest/pages/training-tutorials/nemo-rl-grpo/setup.mdx +++ b/fern/versions/latest/pages/training-tutorials/nemo-rl-grpo/setup.mdx @@ -228,7 +228,7 @@ Prepare the data: ```bash gym dataset collate \ --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ - --resource-server workplace_assistant \ + --resources-server workplace_assistant \ --output-dir data/workplace_assistant \ --mode train_preparation \ --download \ diff --git a/fern/versions/latest/pages/training-tutorials/verl.mdx b/fern/versions/latest/pages/training-tutorials/verl.mdx index ccef3cdfa3..e91f43bc74 100644 --- a/fern/versions/latest/pages/training-tutorials/verl.mdx +++ b/fern/versions/latest/pages/training-tutorials/verl.mdx @@ -37,7 +37,7 @@ cd $NEMO_GYM_ROOT source .venv/bin/activate gym dataset collate \ - --resource-server workplace_assistant \ + --resources-server workplace_assistant \ --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ --output-dir data/workplace_assistant \ --mode train_preparation \ diff --git a/nemo_gym/cli/eval.py b/nemo_gym/cli/eval.py index f6ef237002..1a8644ae50 100644 --- a/nemo_gym/cli/eval.py +++ b/nemo_gym/cli/eval.py @@ -73,7 +73,7 @@ def _fuzzy_matches(query: str, *fields: str) -> bool: def _benchmark_extras(bench: BenchmarkConfig) -> tuple[str, list[str]]: """Resolve a benchmark's config to its `(domain, extra search terms)`. - `BenchmarkConfig` flattens away the resource server name, the resource server `domain`, and the + `BenchmarkConfig` flattens away the resources server name, the resources server `domain`, and the dataset names. We re-resolve the config with the same parser `BenchmarkConfig` uses (so chained `config_paths` / `_inherit_from` are applied) and read those fields back out for the domain column and richer `gym search` matching. @@ -92,10 +92,10 @@ def _benchmark_extras(bench: BenchmarkConfig) -> tuple[str, list[str]]: if not isinstance(instance, (dict, DictConfig)): continue - resource_servers = instance.get("resources_servers") - if resource_servers: + resources_servers = instance.get("resources_servers") + if resources_servers: terms.append(instance_name) # e.g. aime24_math_with_judge_resources_server - for rs_name, rs_config in resource_servers.items(): + for rs_name, rs_config in resources_servers.items(): terms.append(rs_name) # e.g. math_with_judge found_domain = (rs_config or {}).get("domain") if found_domain: @@ -134,7 +134,7 @@ def list_benchmarks() -> None: extras = {name: _benchmark_extras(bench) for name, bench in benchmarks.items()} # `gym search ` reuses this command, narrowing the listing to fuzzy matches across the - # benchmark name, agent name, resource server name, dataset names, and domain. + # benchmark name, agent name, resources server name, dataset names, and domain. query = global_config_dict.get(QUERY_KEY_NAME) if query: benchmarks = { diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index baf191c9c6..29b8786f50 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -127,11 +127,11 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: MODEL_URL = _value_flag("model-url", "policy_base_url", "Model server base URL.") MODEL_API_KEY = _value_flag("model-api-key", "policy_api_key", "Model server API key.") -# Shared flag: select a single resource server by name. Reused by `env test`, `env init`, and `env packages`. -RESOURCE_SERVER = Flag( - register=lambda p: p.add_argument("--resource-server", metavar="NAME", help="Name of the resource server."), +# Shared flag: select a single resources server by name. Reused by `env test`, `env init`, and `env packages`. +RESOURCES_SERVER = Flag( + register=lambda p: p.add_argument("--resources-server", metavar="NAME", help="Name of the resources server."), translate_to_hydra=lambda args: ( - [f"+entrypoint=resources_servers/{args.resource_server}"] if args.resource_server else [] + [f"+entrypoint=resources_servers/{args.resources_server}"] if args.resources_server else [] ), ) @@ -150,7 +150,7 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: # resolving to `//[/].yaml`. A None default flavor falls back to the server name. _ASSETS = { "benchmark": ("benchmarks", "", "config"), - "resource-server": ("resources_servers", "configs", None), + "resources-server": ("resources_servers", "configs", None), "model-type": ("responses_api_models", "configs", None), } @@ -223,7 +223,7 @@ def _asset_selector(flag: str) -> Flag: BENCHMARK = _asset_selector("benchmark") -RESOURCE_SERVER_CONFIG = _asset_selector("resource-server") +RESOURCES_SERVER_CONFIG = _asset_selector("resources-server") MODEL_TYPE = _asset_selector("model-type") # Shared flag: register extra root dirs to search for named components. Consumed by the asset selectors above @@ -258,7 +258,7 @@ def _eval_run(args: argparse.Namespace, overrides: list[str]) -> None: def _env_test(args: argparse.Namespace, overrides: list[str]) -> None: # Run a single server's tests if +entrypoint was passed. No need to check for - # --resource-server because it is translated to +entrypoint in the flag definition. + # --resources-server because it is translated to +entrypoint in the flag definition. has_entrypoint = any(override.lstrip("+").split("=", 1)[0] == "entrypoint" for override in overrides) dispatch("nemo_gym.cli.env:test" if has_entrypoint else "nemo_gym.cli.env:test_all", overrides) @@ -369,7 +369,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: summary="Validate and collate the dataset.", flags=( CONFIG, - RESOURCE_SERVER_CONFIG, + RESOURCES_SERVER_CONFIG, SEARCH_DIR, _value_flag("mode", "mode", "Data preparation mode.", choices=("train_preparation", "example_validation")), _value_flag("output-dir", "output_dirpath", "Output directory for the prepared data."), @@ -379,7 +379,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "env init": Command( target="nemo_gym.cli.env:init_resources_server", summary="Scaffold config for a new server, benchmark, or agent.", - flags=(RESOURCE_SERVER,), + flags=(RESOURCES_SERVER,), ), "env resolve": Command( target="nemo_gym.cli.env:dump_config", @@ -388,9 +388,9 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: ), "env packages": Command( target="nemo_gym.cli.env:pip_list", - summary="Print pip packages for the selected resource server.", + summary="Print pip packages for the selected resources server.", flags=( - RESOURCE_SERVER, + RESOURCES_SERVER, _bool_flag("outdated", "outdated", "List only outdated packages."), Flag( register=lambda p: p.add_argument( @@ -402,13 +402,13 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: ), "env test": Command( target=_env_test, - summary="Test the resource server(s); runs all if no resource server is given.", - flags=(RESOURCE_SERVER,), + summary="Test the resources server(s); runs all if no resources server is given.", + flags=(RESOURCES_SERVER,), ), "env run": Command( target="nemo_gym.cli.env:run", summary="Start the servers.", - flags=(CONFIG, BENCHMARK, RESOURCE_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, MODEL, MODEL_URL, MODEL_API_KEY), + flags=(CONFIG, BENCHMARK, RESOURCES_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, MODEL, MODEL_URL, MODEL_API_KEY), ), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status.", flags=(JSON,)), "eval prepare": Command( @@ -422,7 +422,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: flags=( CONFIG, BENCHMARK, - RESOURCE_SERVER_CONFIG, + RESOURCES_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, Flag( diff --git a/resources_servers/abstention/README.md b/resources_servers/abstention/README.md index a51aa62f4a..eb52885379 100644 --- a/resources_servers/abstention/README.md +++ b/resources_servers/abstention/README.md @@ -17,7 +17,7 @@ The dataset used is [HotPotQA](https://hotpotqa.github.io/) (fullwiki split). ```bash gym env run \ --model-type openai_model \ - --resource-server abstention \ + --resources-server abstention \ +abstention.resources_servers.abstention.judge_model_server.name=policy_model ``` diff --git a/resources_servers/arc_agi/README.md b/resources_servers/arc_agi/README.md index aea6824f12..0347501602 100644 --- a/resources_servers/arc_agi/README.md +++ b/resources_servers/arc_agi/README.md @@ -52,7 +52,7 @@ uv sync ### Start ARC-AGI environment (we can reuse the same one for ARC-AGI-1 and 2): ```bash gym env run \ - --resource-server arc_agi \ + --resources-server arc_agi \ --model-type vllm_model ``` diff --git a/resources_servers/arena_judge/README.md b/resources_servers/arena_judge/README.md index 14a8b6865e..406c5e909b 100644 --- a/resources_servers/arena_judge/README.md +++ b/resources_servers/arena_judge/README.md @@ -49,7 +49,7 @@ Each JSONL row must carry the following top-level fields (pydantic # Running servers gym env run \ --model-type vllm_model \ - --resource-server arena_judge + --resources-server arena_judge # Collecting rollouts (5-example smoke test) gym eval run --no-serve \ diff --git a/resources_servers/asr_with_pc/README.md b/resources_servers/asr_with_pc/README.md index 17e8521715..58c2019724 100644 --- a/resources_servers/asr_with_pc/README.md +++ b/resources_servers/asr_with_pc/README.md @@ -52,7 +52,7 @@ workaround until the schema is extended. ```bash gym env run \ --model-type vllm_model \ - --resource-server asr_with_pc + --resources-server asr_with_pc ``` ## Collecting rollouts (5-example smoke test) diff --git a/resources_servers/aviary/README.md b/resources_servers/aviary/README.md index 4f062ac57c..b08a9e10fc 100644 --- a/resources_servers/aviary/README.md +++ b/resources_servers/aviary/README.md @@ -23,7 +23,7 @@ Run the GSM8K Aviary resources server together with a model config: ```bash gym env run \ - --resource-server aviary/gsm8k_aviary \ + --resources-server aviary/gsm8k_aviary \ --model-type vllm_model ``` @@ -50,7 +50,7 @@ Once the dataset server is running and is accessible at a specific URL, update y ```bash gym env run \ - --resource-server aviary/bbh_remote \ + --resources-server aviary/bbh_remote \ --model-type vllm_model ``` @@ -101,7 +101,7 @@ cd /path/to/gym/directory And then bring up NeMo-Gym: ```bash gym env run \ - --resource-server aviary/bbh_bundled \ + --resources-server aviary/bbh_bundled \ --model-type vllm_model ``` ```bash diff --git a/resources_servers/bigcodebench/README.md b/resources_servers/bigcodebench/README.md index bd21b9f18e..d35f44967a 100644 --- a/resources_servers/bigcodebench/README.md +++ b/resources_servers/bigcodebench/README.md @@ -25,7 +25,7 @@ starts are instant. # Running servers gym env run \ --model-type vllm_model \ - --resource-server bigcodebench + --resources-server bigcodebench # Collecting rollouts (5-example smoke test) gym eval run --no-serve \ diff --git a/resources_servers/bird_sql/README.md b/resources_servers/bird_sql/README.md index eccb92d2f9..d8038466e0 100644 --- a/resources_servers/bird_sql/README.md +++ b/resources_servers/bird_sql/README.md @@ -36,7 +36,7 @@ ensure_bird_sql() ```bash gym env run \ --model-type vllm_model \ - --resource-server bird_sql + --resources-server bird_sql ``` Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in diff --git a/resources_servers/blackjack/README.md b/resources_servers/blackjack/README.md index 927f6bc7dc..caf8af8f2b 100644 --- a/resources_servers/blackjack/README.md +++ b/resources_servers/blackjack/README.md @@ -10,7 +10,7 @@ Example data provided in `data/example.jsonl` (system prompt only, no verifier_m ```bash gym env run \ - --resource-server blackjack \ + --resources-server blackjack \ --model-type vllm_model ``` diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index f9254c72fa..407c81ffe9 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -30,7 +30,7 @@ judge_model_name: Qwen/Qwen3-235B-A22B-Instruct-2507 ```bash # If you want to run with browsecomp benchmark instead of the example samples, need to change the datasets part to the one like `benchmarks/browsecomp/config.yaml` in `resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml` gym env run \ - --resource-server browsecomp_advanced_harness \ + --resources-server browsecomp_advanced_harness \ --model-type vllm_model ``` diff --git a/resources_servers/calendar/README.md b/resources_servers/calendar/README.md index c20b933755..34279ea4e5 100644 --- a/resources_servers/calendar/README.md +++ b/resources_servers/calendar/README.md @@ -18,7 +18,7 @@ The following is an example command for running this resources server along with ```bash gym env run \ --model-type openai_model \ - --resource-server calendar + --resources-server calendar ``` ## Collecting rollouts diff --git a/resources_servers/circle_click/README.md b/resources_servers/circle_click/README.md index ea751da7f6..53815c8a76 100644 --- a/resources_servers/circle_click/README.md +++ b/resources_servers/circle_click/README.md @@ -13,7 +13,7 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & gym env run \ - --resource-server circle_click \ + --resources-server circle_click \ --model-type vllm_model & gym eval run --no-serve \ --agent circle_click_simple_agent \ diff --git a/resources_servers/circle_count/README.md b/resources_servers/circle_count/README.md index e77665e9bd..72f6c87352 100644 --- a/resources_servers/circle_count/README.md +++ b/resources_servers/circle_count/README.md @@ -20,7 +20,7 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & gym env run \ - --resource-server circle_count \ + --resources-server circle_count \ --model-type vllm_model & gym eval run --no-serve \ --agent circle_count_simple_agent \ diff --git a/resources_servers/code_fim/README.md b/resources_servers/code_fim/README.md index 2415e649e9..e55c4595dd 100644 --- a/resources_servers/code_fim/README.md +++ b/resources_servers/code_fim/README.md @@ -60,7 +60,7 @@ the reward; pass@k / majority@k are computed by the metrics layer. # Running servers gym env run \ --model-type vllm_model \ - --resource-server code_fim + --resources-server code_fim # Collecting rollouts (5-example smoke test) gym eval run --no-serve \ diff --git a/resources_servers/code_gen/README.md b/resources_servers/code_gen/README.md index 71ec0cd132..7af636f064 100644 --- a/resources_servers/code_gen/README.md +++ b/resources_servers/code_gen/README.md @@ -26,7 +26,7 @@ We use the LiveCodeBench execution code. # Running the server gym env run \ --model-type openai_model \ - --resource-server code_gen + --resources-server code_gen # Collect rollouts from example problems gym eval run --no-serve \ diff --git a/resources_servers/cvdp/README.md b/resources_servers/cvdp/README.md index 3251871b55..e6c3320b58 100644 --- a/resources_servers/cvdp/README.md +++ b/resources_servers/cvdp/README.md @@ -124,7 +124,7 @@ apt install -y ./apptainer_1.3.1_amd64.deb ```bash gym env run \ - --resource-server cvdp \ + --resources-server cvdp \ --model-type vllm_model ``` @@ -142,7 +142,7 @@ gym eval run --no-serve \ --max-output-tokens 4096 \ --temperature 0.2 \ --top-p 0.7 \ - --resource-server cvdp \ + --resources-server cvdp \ --model-type vllm_model \ --resume ``` diff --git a/resources_servers/equivalence_llm_judge/README.md b/resources_servers/equivalence_llm_judge/README.md index 900a562833..e4c0974b69 100644 --- a/resources_servers/equivalence_llm_judge/README.md +++ b/resources_servers/equivalence_llm_judge/README.md @@ -52,7 +52,7 @@ equivalence_llm_judge_simple_agent: Spin up with a judge model and prompt: ```bash gym env run \ - --resource-server equivalence_llm_judge \ + --resources-server equivalence_llm_judge \ --model-type openai_model \ +equivalence_llm_judge.resources_servers.equivalence_llm_judge.judge_responses_create_params.max_output_tokens=256 \ +equivalence_llm_judge.resources_servers.equivalence_llm_judge.judge_system_message="You are a careful arbiter." \ diff --git a/resources_servers/ether0/README.md b/resources_servers/ether0/README.md index c4fab26603..8ab3d56419 100644 --- a/resources_servers/ether0/README.md +++ b/resources_servers/ether0/README.md @@ -28,7 +28,7 @@ Start servers and collect rollouts # start vllm and nemo gym servers vllm serve futurehouse/ether0 & gym env run \ - --resource-server ether0 \ + --resources-server ether0 \ --model-type vllm_model & # wait for above to be ready diff --git a/resources_servers/evalplus/README.md b/resources_servers/evalplus/README.md index 5024e5c09b..64421b3b5a 100644 --- a/resources_servers/evalplus/README.md +++ b/resources_servers/evalplus/README.md @@ -48,7 +48,7 @@ compute pass@k for each separately. # Running servers gym env run \ --model-type vllm_model \ - --resource-server evalplus + --resources-server evalplus # Collecting rollouts (5-example smoke test) gym eval run --no-serve \ diff --git a/resources_servers/example_multi_turn_gymnasium/README.md b/resources_servers/example_multi_turn_gymnasium/README.md index 49cd5bf965..2b7bc95d29 100644 --- a/resources_servers/example_multi_turn_gymnasium/README.md +++ b/resources_servers/example_multi_turn_gymnasium/README.md @@ -16,7 +16,7 @@ Example data provided in `data/example.jsonl`. ```bash gym env run \ - --resource-server example_multi_turn_gymnasium \ + --resources-server example_multi_turn_gymnasium \ --model-type vllm_model ``` diff --git a/resources_servers/finance_sec_search/README.md b/resources_servers/finance_sec_search/README.md index 3518779796..9ecbfd5ecb 100644 --- a/resources_servers/finance_sec_search/README.md +++ b/resources_servers/finance_sec_search/README.md @@ -196,7 +196,7 @@ With a local vLLM model server: ```bash gym env run \ --model-type vllm_model \ - --resource-server finance_sec_search + --resources-server finance_sec_search ``` Or with an OpenAI-compatible API (e.g. OpenAI, Azure, NIM): @@ -204,7 +204,7 @@ Or with an OpenAI-compatible API (e.g. OpenAI, Azure, NIM): ```bash gym env run \ --model-type openai_model \ - --resource-server finance_sec_search + --resources-server finance_sec_search ``` ### 4. Collect rollouts @@ -229,7 +229,7 @@ gym eval run --no-serve \ ### Run tests ```bash -gym env test --resource-server finance_sec_search +gym env test --resources-server finance_sec_search ``` ## Verification diff --git a/resources_servers/format_verification/README.md b/resources_servers/format_verification/README.md index 3ee6df91c0..78c9d48921 100644 --- a/resources_servers/format_verification/README.md +++ b/resources_servers/format_verification/README.md @@ -28,7 +28,7 @@ See [ARCHITECTURE.md](ARCHITECTURE.md) for system diagrams and detailed field ma ```bash gym env run \ --model-type openai_model \ - --resource-server format_verification/freeform_formatting + --resources-server format_verification/freeform_formatting ``` Collect rollouts: @@ -45,7 +45,7 @@ gym eval run --no-serve \ ```bash gym env run \ --model-type openai_model \ - --resource-server format_verification/citation_format + --resources-server format_verification/citation_format ``` Collect rollouts: @@ -64,7 +64,7 @@ gym eval run --no-serve \ ```bash gym dataset collate \ --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ - --resource-server format_verification/freeform_formatting \ + --resources-server format_verification/freeform_formatting \ --output-dir data/format_verification_freeform/ \ --mode train_preparation \ --download @@ -74,7 +74,7 @@ gym dataset collate \ ```bash gym dataset collate \ --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ - --resource-server format_verification/citation_format \ + --resources-server format_verification/citation_format \ --output-dir data/format_verification_citation/ \ --mode train_preparation \ --download @@ -82,7 +82,7 @@ gym dataset collate \ ## Testing ```bash -gym env test --resource-server format_verification +gym env test --resources-server format_verification ``` ## Licensing diff --git a/resources_servers/frontierscience_judge/README.md b/resources_servers/frontierscience_judge/README.md index 6cd232151d..00823b48e3 100644 --- a/resources_servers/frontierscience_judge/README.md +++ b/resources_servers/frontierscience_judge/README.md @@ -37,7 +37,7 @@ judge's full text), and in research mode `rubric_score` plus # Running servers gym env run \ --model-type vllm_model \ - --resource-server frontierscience_judge + --resources-server frontierscience_judge # Collecting rollouts (5-example smoke test) gym eval run --no-serve \ diff --git a/resources_servers/google_search/README.md b/resources_servers/google_search/README.md index 79c029da86..fb8202c1ec 100644 --- a/resources_servers/google_search/README.md +++ b/resources_servers/google_search/README.md @@ -100,7 +100,7 @@ tools=[ ```bash gym env run \ --model-type openai_model \ - --resource-server google_search + --resources-server google_search gym eval run --no-serve \ --agent simple_agent \ diff --git a/resources_servers/gpqa_diamond/README.md b/resources_servers/gpqa_diamond/README.md index 9b939d412d..7530ed2d54 100644 --- a/resources_servers/gpqa_diamond/README.md +++ b/resources_servers/gpqa_diamond/README.md @@ -80,7 +80,7 @@ Using a local Nemotron 3 model with `local_vllm_model`: gym env run \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type local_vllm_model/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 \ - --resource-server gpqa_diamond \ + --resources-server gpqa_diamond \ '++policy_model=${inherit_from:NVIDIA-Nemotron-3-Nano-30B-A3B-BF16}' \ +simple_agent.responses_api_agents.simple_agent.resources_server.name=gpqa_diamond \ ++NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.responses_api_models.local_vllm_model.vllm_serve_kwargs.mamba_ssm_cache_dtype=float32 \ @@ -93,7 +93,7 @@ Generic example with `openai_model`: gym env run \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type openai_model \ - --resource-server gpqa_diamond \ + --resources-server gpqa_diamond \ +simple_agent.responses_api_agents.simple_agent.resources_server.name=gpqa_diamond gym eval run --no-serve \ diff --git a/resources_servers/graphwalks/README.md b/resources_servers/graphwalks/README.md index 8ccfcfbc46..579e39efe0 100644 --- a/resources_servers/graphwalks/README.md +++ b/resources_servers/graphwalks/README.md @@ -26,7 +26,7 @@ https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/evaluation/evaluator ```bash gym env run \ - --resource-server graphwalks \ + --resources-server graphwalks \ --model-type vllm_model ``` diff --git a/resources_servers/grl_sokoban/README.md b/resources_servers/grl_sokoban/README.md index 38087c2a5f..f562787ce8 100644 --- a/resources_servers/grl_sokoban/README.md +++ b/resources_servers/grl_sokoban/README.md @@ -14,7 +14,7 @@ Spin up the server alongside a compatible agent: gym env run \ --model-type openai_model \ --config responses_api_agents/gymnasium_agent/configs/gymnasium_agent.yaml \ - --resource-server grl_sokoban + --resources-server grl_sokoban ``` Collect trajectories: diff --git a/resources_servers/grl_tetris/README.md b/resources_servers/grl_tetris/README.md index 0b1abb283d..a2b469814b 100644 --- a/resources_servers/grl_tetris/README.md +++ b/resources_servers/grl_tetris/README.md @@ -15,7 +15,7 @@ Start NeMo Gym servers gym env run \ --model-type openai_model \ --config responses_api_agents/gymnasium_agent/configs/gymnasium_agent.yaml \ - --resource-server grl_tetris + --resources-server grl_tetris ``` Collect trajectories: diff --git a/resources_servers/hotpotqa_qa/README.md b/resources_servers/hotpotqa_qa/README.md index 94a1ad4b51..2161aa182a 100644 --- a/resources_servers/hotpotqa_qa/README.md +++ b/resources_servers/hotpotqa_qa/README.md @@ -32,7 +32,7 @@ pass@k for each channel. ```bash gym env run \ --model-type openai_model \ - --resource-server hotpotqa_qa + --resources-server hotpotqa_qa ``` ## Collecting rollouts diff --git a/resources_servers/imo_gradingbench/README.md b/resources_servers/imo_gradingbench/README.md index 05e6bb9ccc..6e357071b3 100644 --- a/resources_servers/imo_gradingbench/README.md +++ b/resources_servers/imo_gradingbench/README.md @@ -70,7 +70,7 @@ grade. # Running servers gym env run \ --model-type vllm_model \ - --resource-server imo_gradingbench + --resources-server imo_gradingbench # Collecting rollouts (5-example smoke test) gym eval run --no-serve \ diff --git a/resources_servers/imo_proofbench_judge/README.md b/resources_servers/imo_proofbench_judge/README.md index 9dab8bfd01..9bc96b93d4 100644 --- a/resources_servers/imo_proofbench_judge/README.md +++ b/resources_servers/imo_proofbench_judge/README.md @@ -24,7 +24,7 @@ Use ``benchmarks/imo_proofbench/`` for the IMO-ProofBench dataset. ```bash gym env run \ --model-type vllm_model \ - --resource-server imo_proofbench_judge \ + --resources-server imo_proofbench_judge \ +judge_base_url=https://generativelanguage.googleapis.com/v1beta/openai \ "+judge_api_key=$GEMINI_API_KEY" \ +judge_model_name=gemini-2.5-pro diff --git a/resources_servers/instruction_following/README.md b/resources_servers/instruction_following/README.md index 5d266ad526..9b99029921 100644 --- a/resources_servers/instruction_following/README.md +++ b/resources_servers/instruction_following/README.md @@ -50,7 +50,7 @@ The dataset can be found at https://huggingface.co/datasets/nvidia/Nemotron-RL-i gym env run \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type openai_model \ - --resource-server instruction_following \ + --resources-server instruction_following \ +simple_agent.responses_api_agents.simple_agent.resources_server.name=instruction_following gym dataset download --storage gitlab \ @@ -83,7 +83,7 @@ We run reward profiling with Qwen/Qwen3-30B-A3B-Instruct-2507. We evaluate the m ## Testing ```bash -gym env test --resource-server instruction_following +gym env test --resources-server instruction_following ``` diff --git a/resources_servers/inverse_if/README.md b/resources_servers/inverse_if/README.md index a69e0044a5..88bd7f8f12 100644 --- a/resources_servers/inverse_if/README.md +++ b/resources_servers/inverse_if/README.md @@ -6,11 +6,11 @@ Evaluates model responses on the **Inverse IF** (Instruction Following) benchmar ```bash # 1. Run unit tests -gym env test --resource-server inverse_if +gym env test --resources-server inverse_if # 2. Start servers (in terminal 1) gym env run \ - --resource-server inverse_if \ + --resources-server inverse_if \ --model-type vllm_model # 3. Collect rollouts on example data (in terminal 2) @@ -101,7 +101,7 @@ gym eval run --no-serve \ ```bash # Run all unit tests -gym env test --resource-server inverse_if +gym env test --resources-server inverse_if # Or run directly with pytest cd resources_servers/inverse_if @@ -119,7 +119,7 @@ Tests cover: 1. **Start servers**: ```bash gym env run \ - --resource-server inverse_if \ + --resources-server inverse_if \ --model-type vllm_model ``` diff --git a/resources_servers/jailbreak_detection/README.md b/resources_servers/jailbreak_detection/README.md index b857e7481d..86f827d005 100644 --- a/resources_servers/jailbreak_detection/README.md +++ b/resources_servers/jailbreak_detection/README.md @@ -28,17 +28,17 @@ The example dataset includes various jailbreak attack patterns: 1. Start the servers: ```bash gym env run \ - --resource-server jailbreak_detection/jailbreak_detection_nemotron_combined_reward_tp8 \ + --resources-server jailbreak_detection/jailbreak_detection_nemotron_combined_reward_tp8 \ --model-type openai_model \ - --resource-server jailbreak_detection/safety_judge_model + --resources-server jailbreak_detection/safety_judge_model ``` 2. Collect rollouts: ```bash gym eval run --no-serve \ - --resource-server jailbreak_detection/jailbreak_detection_nemotron_combined_reward_tp8 \ + --resources-server jailbreak_detection/jailbreak_detection_nemotron_combined_reward_tp8 \ --model-type openai_model \ - --resource-server jailbreak_detection/safety_judge_model \ + --resources-server jailbreak_detection/safety_judge_model \ --agent jailbreak_detection_simple_agent \ --input resources_servers/jailbreak_detection/data/example.jsonl \ --output results/jailbreak_detection_rollouts.jsonl diff --git a/resources_servers/labbench2_vlm/README.md b/resources_servers/labbench2_vlm/README.md index 2c2fe07bbc..afe5c34bb7 100644 --- a/resources_servers/labbench2_vlm/README.md +++ b/resources_servers/labbench2_vlm/README.md @@ -179,8 +179,8 @@ Start the servers: ```bash gym env run \ - --resource-server labbench2_vlm \ - --resource-server labbench2_vlm/judge_model_openai \ + --resources-server labbench2_vlm \ + --resources-server labbench2_vlm/judge_model_openai \ --model-type openai_model ``` @@ -188,8 +188,8 @@ Collect rollouts: ```bash gym eval run --no-serve \ - --resource-server labbench2_vlm \ - --resource-server labbench2_vlm/judge_model_openai \ + --resources-server labbench2_vlm \ + --resources-server labbench2_vlm/judge_model_openai \ --model-type openai_model \ --agent labbench2_vlm_simple_agent \ --input resources_servers/labbench2_vlm/data/figqa2_img_validation.jsonl \ @@ -208,8 +208,8 @@ because `data/test_media/` and `data/example.jsonl` are committed): ```bash gym eval run --no-serve \ - --resource-server labbench2_vlm \ - --resource-server labbench2_vlm/judge_model_openai \ + --resources-server labbench2_vlm \ + --resources-server labbench2_vlm/judge_model_openai \ --model-type openai_model \ --agent labbench2_vlm_simple_agent \ --input resources_servers/labbench2_vlm/data/example.jsonl \ @@ -230,8 +230,8 @@ agent ref are auto-derived from the dataset entry in the chained config ```bash gym eval run \ - --resource-server labbench2_vlm \ - --resource-server labbench2_vlm/judge_model_openai \ + --resources-server labbench2_vlm \ + --resources-server labbench2_vlm/judge_model_openai \ --model-type openai_model \ --split validation \ --output results/labbench2_vlm_validation.jsonl \ diff --git a/resources_servers/longmt_eval/README.md b/resources_servers/longmt_eval/README.md index 3bf4e73582..f35cfe6b2b 100644 --- a/resources_servers/longmt_eval/README.md +++ b/resources_servers/longmt_eval/README.md @@ -159,7 +159,7 @@ rollouts file. ```bash # Start servers (smoke-test mode — no GPU needed for the verifier) gym env run \ - --resource-server longmt_eval \ + --resources-server longmt_eval \ --model-type vllm_model & \ ++longmt_eval.resources_servers.longmt_eval.compute_segale=false diff --git a/resources_servers/math_advanced_calculations/README.md b/resources_servers/math_advanced_calculations/README.md index eed8c1581a..fa4593d88d 100644 --- a/resources_servers/math_advanced_calculations/README.md +++ b/resources_servers/math_advanced_calculations/README.md @@ -10,7 +10,7 @@ Spin up server: ``` gym env run \ --model-type openai_model \ - --resource-server math_advanced_calculations + --resources-server math_advanced_calculations ``` Collect trajectories: diff --git a/resources_servers/math_proof_judgement/README.md b/resources_servers/math_proof_judgement/README.md index 10d8a9183a..2c9aca9040 100644 --- a/resources_servers/math_proof_judgement/README.md +++ b/resources_servers/math_proof_judgement/README.md @@ -68,7 +68,7 @@ parser name that matches your model's reasoning tokens (see `vllm serve # Running servers gym env run \ --model-type vllm_model \ - --resource-server math_proof_judgement + --resources-server math_proof_judgement # Collecting rollouts (5-example smoke test) gym eval run --no-serve \ diff --git a/resources_servers/math_with_autograder/README.md b/resources_servers/math_with_autograder/README.md index d842b3818e..aa310e9cd9 100644 --- a/resources_servers/math_with_autograder/README.md +++ b/resources_servers/math_with_autograder/README.md @@ -37,8 +37,8 @@ existing math_with_judge consumers. ```bash gym env run \ --model-type vllm_model \ - --resource-server math_with_autograder \ - --resource-server math_with_autograder/judge_gptoss20b + --resources-server math_with_autograder \ + --resources-server math_with_autograder/judge_gptoss20b ``` The bundled `judge_gptoss20b.yaml` wires the autograder judge to diff --git a/resources_servers/math_with_code/README.md b/resources_servers/math_with_code/README.md index 17f4d43dd4..aab5e33323 100644 --- a/resources_servers/math_with_code/README.md +++ b/resources_servers/math_with_code/README.md @@ -108,7 +108,7 @@ Start server gym env run \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type openai_model \ - --resource-server math_with_code \ + --resources-server math_with_code \ +simple_agent.responses_api_agents.simple_agent.resources_server.name=math_with_code ``` diff --git a/resources_servers/math_with_judge/README.md b/resources_servers/math_with_judge/README.md index 46366149bc..50bb775348 100644 --- a/resources_servers/math_with_judge/README.md +++ b/resources_servers/math_with_judge/README.md @@ -13,7 +13,7 @@ The following are example commands for running this resources server, along with ```bash gym env run \ --model-type openai_model \ - --resource-server math_with_judge \ + --resources-server math_with_judge \ +math_with_judge.resources_servers.math_with_judge.judge_model_server.name=policy_model ``` diff --git a/resources_servers/mcqa/README.md b/resources_servers/mcqa/README.md index 68f4c4be41..8b278f7ffb 100644 --- a/resources_servers/mcqa/README.md +++ b/resources_servers/mcqa/README.md @@ -114,7 +114,7 @@ For datasets with custom prompt formats, you can optionally use `template_metada gym env run \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type openai_model \ - --resource-server mcqa \ + --resources-server mcqa \ +simple_agent.responses_api_agents.simple_agent.resources_server.name=mcqa gym eval run --no-serve \ diff --git a/resources_servers/mrcr/README.md b/resources_servers/mrcr/README.md index 093fe529ff..18c138e6a7 100644 --- a/resources_servers/mrcr/README.md +++ b/resources_servers/mrcr/README.md @@ -28,7 +28,7 @@ always fails. ```bash gym env run \ - --resource-server mrcr \ + --resources-server mrcr \ --model-type vllm_model ``` diff --git a/resources_servers/multichallenge/README.md b/resources_servers/multichallenge/README.md index 7c9a4465ad..b475e6ff4b 100644 --- a/resources_servers/multichallenge/README.md +++ b/resources_servers/multichallenge/README.md @@ -6,11 +6,11 @@ Evaluates model responses on the **MultiChallenge** benchmark using an LLM judge ```bash # 1. Run unit tests -gym env test --resource-server multichallenge +gym env test --resources-server multichallenge # 2. Start servers (in terminal 1) gym env run \ - --resource-server multichallenge \ + --resources-server multichallenge \ --model-type vllm_model # 3. Collect rollouts on example data (in terminal 2) @@ -89,7 +89,7 @@ gym eval run --no-serve \ ```bash # Run all unit tests -gym env test --resource-server multichallenge +gym env test --resources-server multichallenge # Or run directly with pytest for more detail cd resources_servers/multichallenge @@ -107,7 +107,7 @@ Tests cover: 1. **Start servers**: ```bash gym env run \ - --resource-server multichallenge \ + --resources-server multichallenge \ --model-type vllm_model ``` diff --git a/resources_servers/newton_bench/README.md b/resources_servers/newton_bench/README.md index c8359dc7a9..4e4228d5df 100644 --- a/resources_servers/newton_bench/README.md +++ b/resources_servers/newton_bench/README.md @@ -92,7 +92,7 @@ vllm serve \ ### Launch servers ```bash gym env run \ - --resource-server newton_bench \ + --resources-server newton_bench \ --model-type vllm_model ``` @@ -107,7 +107,7 @@ gym eval run --no-serve \ ## Running Tests ```bash -gym env test --resource-server newton_bench +gym env test --resources-server newton_bench ``` ## Qwen/Qwen3-VL-8B-Thinking Evaluation Summary diff --git a/resources_servers/ns_tools/README.md b/resources_servers/ns_tools/README.md index b2137d933e..47d2999738 100644 --- a/resources_servers/ns_tools/README.md +++ b/resources_servers/ns_tools/README.md @@ -9,8 +9,8 @@ It integrates with the NeMo Skills ToolManager to dynamically load and execute t The following are example commands for running this resources server with the simple agent and a vLLM model: ```bash gym env run \ - --resource-server ns_tools \ - --resource-server math_with_judge \ + --resources-server ns_tools \ + --resources-server math_with_judge \ --model-type vllm_model \ --model-url http://localhost:8000/v1 \ --model Qwen/Qwen3-8B \ @@ -79,7 +79,7 @@ python prepare_dataset.py \ To validate the example data and regenerate metrics: ```bash gym dataset collate \ - --resource-server ns_tools \ + --resources-server ns_tools \ --config responses_api_models/openai_model/configs/openai_model.yaml \ --output-dir data/ns_tools \ --mode example_validation diff --git a/resources_servers/openenv/README.md b/resources_servers/openenv/README.md index 22032cfba4..158dc62a5b 100644 --- a/resources_servers/openenv/README.md +++ b/resources_servers/openenv/README.md @@ -36,7 +36,7 @@ policy_model_name: Running an environment is a **two-step process**: -1. **Start the servers** with `gym env run` — this launches the resource server, agent server, and model server. Wait until you see `All 3 / 3 servers ready!`. +1. **Start the servers** with `gym env run` — this launches the resources server, agent server, and model server. Wait until you see `All 3 / 3 servers ready!`. 2. **Collect rollouts** with `gym eval run --no-serve` in a **separate terminal** — this sends the JSONL prompts through the agent, which calls the LLM and environment in a loop, and writes trajectories with rewards to an output file. #### Echo @@ -45,7 +45,7 @@ Running an environment is a **two-step process**: # Terminal 1: Start servers source .venv/bin/activate gym env run \ - --resource-server openenv/openenv_echo \ + --resources-server openenv/openenv_echo \ --model-type openai_model # Terminal 2: Collect rollouts (after "All 3 / 3 servers ready!") @@ -63,7 +63,7 @@ gym eval run --no-serve \ # Terminal 1: Start servers source .venv/bin/activate gym env run \ - --resource-server openenv/openenv_coding \ + --resources-server openenv/openenv_coding \ --model-type openai_model # Terminal 2: Collect rollouts @@ -81,7 +81,7 @@ gym eval run --no-serve \ # Terminal 1: Start servers source .venv/bin/activate gym env run \ - --resource-server openenv/openenv_maze \ + --resources-server openenv/openenv_maze \ --model-type openai_model # Terminal 2: Collect rollouts diff --git a/resources_servers/physics_judge/README.md b/resources_servers/physics_judge/README.md index 8cec2511f9..04e268ae2b 100644 --- a/resources_servers/physics_judge/README.md +++ b/resources_servers/physics_judge/README.md @@ -34,8 +34,8 @@ existing math_with_judge consumers (`aime24`, `aime25`, `gsm8k`, ```bash gym env run \ --model-type vllm_model \ - --resource-server physics_judge \ - --resource-server physics_judge/judge_openai + --resources-server physics_judge \ + --resources-server physics_judge/judge_openai ``` The bundled `judge_openai.yaml` defaults the judge to `openai/gpt-oss-20b` diff --git a/resources_servers/polymath/README.md b/resources_servers/polymath/README.md index 17a20bb0f1..cb2813946e 100644 --- a/resources_servers/polymath/README.md +++ b/resources_servers/polymath/README.md @@ -32,7 +32,7 @@ PolyMath additions are at the metric-aggregation layer: ```bash gym env run \ --model-type vllm_model \ - --resource-server polymath + --resources-server polymath ``` ## Collecting rollouts (5-example smoke test) diff --git a/resources_servers/rdkit_chemistry/README.md b/resources_servers/rdkit_chemistry/README.md index a932ecacba..241d63e9dc 100644 --- a/resources_servers/rdkit_chemistry/README.md +++ b/resources_servers/rdkit_chemistry/README.md @@ -66,7 +66,7 @@ takes precedence over `use_box_format`. ```bash gym env run \ - --resource-server rdkit_chemistry \ + --resources-server rdkit_chemistry \ --model-type openai_model gym eval run --no-serve \ diff --git a/resources_servers/reasoning_gym/README.md b/resources_servers/reasoning_gym/README.md index 691109098c..2dbf312c26 100644 --- a/resources_servers/reasoning_gym/README.md +++ b/resources_servers/reasoning_gym/README.md @@ -83,7 +83,7 @@ policy_model_name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 ```bash gym env run \ - --resource-server reasoning_gym \ + --resources-server reasoning_gym \ --model-type vllm_model ``` diff --git a/resources_servers/simpleqa/README.md b/resources_servers/simpleqa/README.md index c0dedfa747..c600aa0581 100644 --- a/resources_servers/simpleqa/README.md +++ b/resources_servers/simpleqa/README.md @@ -41,7 +41,7 @@ the reasoning trace is split off before the response reaches this server. ```bash gym env run \ --model-type vllm_model \ - --resource-server simpleqa \ + --resources-server simpleqa \ +simpleqa.resources_servers.simpleqa.judge_model_server.name=policy_model ``` diff --git a/resources_servers/single_step_tool_use_with_argument_comparison/README.md b/resources_servers/single_step_tool_use_with_argument_comparison/README.md index adb3c6d98b..c225512251 100644 --- a/resources_servers/single_step_tool_use_with_argument_comparison/README.md +++ b/resources_servers/single_step_tool_use_with_argument_comparison/README.md @@ -9,7 +9,7 @@ Data links: ? The following command can be used to run this resources server, along with the tool simulation agent and an OpenAI model: ```bash gym env run \ - --resource-server single_step_tool_use_with_argument_comparison \ + --resources-server single_step_tool_use_with_argument_comparison \ --model-type openai_model ``` diff --git a/resources_servers/speed_bench/README.md b/resources_servers/speed_bench/README.md index 7c727906e4..defe759d25 100644 --- a/resources_servers/speed_bench/README.md +++ b/resources_servers/speed_bench/README.md @@ -109,7 +109,7 @@ For an EAGLE3 / MTP setup with a paired draft model, see # your own model config to swap targets; just keep the speculative_config block). gym env run \ --model-type local_vllm_model/Qwen/Qwen3-30B-A3B-Instruct-2507-ngram-specdec \ - --resource-server speed_bench \ + --resources-server speed_bench \ --config responses_api_agents/speed_bench_agent/configs/speed_bench_agent.yaml \ +policy_model=Qwen3-30B-A3B-Instruct-2507-ngram-specdec @@ -129,7 +129,7 @@ when starting the upstream `vllm serve` process. ## Tests ```bash -gym env test --resource-server speed_bench +gym env test --resources-server speed_bench ``` The unit tests cover the Prometheus parser, the running-delta math, the diff --git a/resources_servers/spider2_lite/README.md b/resources_servers/spider2_lite/README.md index 9d0b97defd..6cd852814e 100644 --- a/resources_servers/spider2_lite/README.md +++ b/resources_servers/spider2_lite/README.md @@ -79,7 +79,7 @@ Each line must include either `gold_sql` (execute-and-compare) or `gold_result` Validate example data: ```bash gym dataset collate \ - --resource-server spider2_lite \ + --resources-server spider2_lite \ --output-dir /tmp/prepare \ --mode example_validation ``` diff --git a/resources_servers/structeval/README.md b/resources_servers/structeval/README.md index 8d7715a181..9f74e76c1b 100644 --- a/resources_servers/structeval/README.md +++ b/resources_servers/structeval/README.md @@ -30,7 +30,7 @@ reward = 0.2 * render_score + 0.8 * key_validation_score ```bash gym env run \ --model-type vllm_model \ - --resource-server structeval/structeval_nonrenderable + --resources-server structeval/structeval_nonrenderable ``` ### Collecting rollouts @@ -56,7 +56,7 @@ python resources_servers/structeval/misc/prepare_data.py \ ## Testing ```bash -gym env test --resource-server structeval +gym env test --resources-server structeval ``` ## Licensing diff --git a/resources_servers/structured_outputs/README.md b/resources_servers/structured_outputs/README.md index 9f2d449570..24e78ebb85 100644 --- a/resources_servers/structured_outputs/README.md +++ b/resources_servers/structured_outputs/README.md @@ -57,7 +57,7 @@ an OpenAI model: ```bash gym env run \ --model-type openai_model \ - --resource-server structured_outputs/structured_outputs_json + --resources-server structured_outputs/structured_outputs_json ``` Then, text-output rollouts can be collected using a command such as: @@ -76,7 +76,7 @@ emitted function call is the final answer, not an action to execute: ```bash gym env run \ --model-type vllm_model \ - --resource-server structured_outputs/structured_outputs_v4 + --resources-server structured_outputs/structured_outputs_v4 ``` ```bash @@ -100,7 +100,7 @@ You can prepare the data for training with: ```bash gym dataset collate \ --config responses_api_models/openai_model/configs/openai_model.yaml \ - --resource-server structured_outputs/structured_outputs_json \ + --resources-server structured_outputs/structured_outputs_json \ --output-dir data/structured_outputs \ --mode train_preparation \ --download @@ -111,7 +111,7 @@ gym dataset collate \ # prepare gym dataset collate \ --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ - --resource-server structured_outputs/structured_outputs_json_yaml_xml_v1 \ + --resources-server structured_outputs/structured_outputs_json_yaml_xml_v1 \ --output-dir data/structured_outputs/ \ --mode train_preparation \ --download @@ -121,7 +121,7 @@ gym dataset collate \ ```bash gym dataset collate \ --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ - --resource-server structured_outputs/structured_outputs_v3 \ + --resources-server structured_outputs/structured_outputs_v3 \ --output-dir data/structured_outputs_v3/ \ --mode train_preparation \ --download @@ -142,7 +142,7 @@ Prepare the v4 training data with: ```bash gym dataset collate \ --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ - --resource-server structured_outputs/structured_outputs_v4 \ + --resources-server structured_outputs/structured_outputs_v4 \ --output-dir data/structured_outputs_v4_tool_call/ \ --mode train_preparation \ --download @@ -167,7 +167,7 @@ Generation details and CLI examples are in # Testing ``` -gym env test --resource-server structured_outputs +gym env test --resources-server structured_outputs ``` # Licensing information diff --git a/resources_servers/swerl_gen/README.md b/resources_servers/swerl_gen/README.md index 6911dbe0f8..bedc16245a 100644 --- a/resources_servers/swerl_gen/README.md +++ b/resources_servers/swerl_gen/README.md @@ -63,7 +63,7 @@ You need to have singularity installed. Image having singularity: gym env run \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type openai_model \ - --resource-server swerl_gen + --resources-server swerl_gen gym eval run --no-serve \ --agent swerl_gen_simple_agent \ diff --git a/resources_servers/swerl_llm_judge/README.md b/resources_servers/swerl_llm_judge/README.md index a5976891a1..0799fe1182 100644 --- a/resources_servers/swerl_llm_judge/README.md +++ b/resources_servers/swerl_llm_judge/README.md @@ -58,7 +58,7 @@ Notes: gym env run \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type openai_model \ - --resource-server swerl_llm_judge + --resources-server swerl_llm_judge gym eval run --no-serve \ --agent swerl_llm_judge_simple_agent \ diff --git a/resources_servers/tavily_search/README.md b/resources_servers/tavily_search/README.md index ca398a5235..5e92d5a2bf 100644 --- a/resources_servers/tavily_search/README.md +++ b/resources_servers/tavily_search/README.md @@ -40,7 +40,7 @@ gym dataset download --storage gitlab \ --output resources_servers/tavily_search/data/sft_samples/sft_samples_validation.jsonl gym env run \ - --resource-server tavily_search/tavily_search_judge_vllm_model \ + --resources-server tavily_search/tavily_search_judge_vllm_model \ --model-type vllm_model ``` diff --git a/resources_servers/terminus_judge/README.md b/resources_servers/terminus_judge/README.md index 4b97853ba1..019c082f00 100644 --- a/resources_servers/terminus_judge/README.md +++ b/resources_servers/terminus_judge/README.md @@ -33,7 +33,7 @@ The following command can be used to run this resources server, along with the s ```bash gym env run \ - --resource-server terminus_judge \ + --resources-server terminus_judge \ --model-type openai_model \ +terminus_judge_resources_server.resources_servers.terminus_judge.judge_responses_create_params.max_output_tokens=512 ``` diff --git a/resources_servers/text_to_sql/README.md b/resources_servers/text_to_sql/README.md index 45ab4c0373..91be981082 100644 --- a/resources_servers/text_to_sql/README.md +++ b/resources_servers/text_to_sql/README.md @@ -69,7 +69,7 @@ Each data sample should include: ```bash gym env run \ - --resource-server text_to_sql \ + --resources-server text_to_sql \ --model-type openai_model \ +text_to_sql_resources_server.resources_servers.text_to_sql.judge_responses_create_params.max_output_tokens=512 ``` diff --git a/resources_servers/ugphysics_judge/README.md b/resources_servers/ugphysics_judge/README.md index ef61b34397..bfeba8dfa6 100644 --- a/resources_servers/ugphysics_judge/README.md +++ b/resources_servers/ugphysics_judge/README.md @@ -60,7 +60,7 @@ swap it for any other `responses_api_models/*` config that exposes a # Running servers gym env run \ --model-type vllm_model \ - --resource-server ugphysics_judge \ + --resources-server ugphysics_judge \ --config benchmarks/ugphysics/judge_gptoss20b.yaml # Collecting rollouts (5-example smoke test) diff --git a/resources_servers/verifif/README.md b/resources_servers/verifif/README.md index db57187f8d..9e67aefb97 100644 --- a/resources_servers/verifif/README.md +++ b/resources_servers/verifif/README.md @@ -32,7 +32,7 @@ policy_model_name: gpt-5-2025-08-07 # or gpt-4.1-2025-04-14 cd /path/to/Gym source .venv/bin/activate gym env run \ - --resource-server verifif \ + --resources-server verifif \ --model-type openai_model ``` diff --git a/resources_servers/vlm_eval_kit/README.md b/resources_servers/vlm_eval_kit/README.md index ee924cfc81..a18f9539ef 100644 --- a/resources_servers/vlm_eval_kit/README.md +++ b/resources_servers/vlm_eval_kit/README.md @@ -55,7 +55,7 @@ python run.py --verbose \ First run the VLMEvalKit server to install dependencies. ```bash gym env run \ - --resource-server vlm_eval_kit \ + --resources-server vlm_eval_kit \ --model-type openai_model ``` @@ -77,7 +77,7 @@ WANDB_PROJECT= EXPERIMENT_NAME=vlmevalkit/gpt-4o-mini-20240718 gym eval run \ --model-type openai_model \ - --resource-server vlm_eval_kit \ + --resources-server vlm_eval_kit \ --output results/$EXPERIMENT_NAME.jsonl \ --split validation \ --model gpt-4o-mini-2024-07-18 \ diff --git a/resources_servers/wmt_translation/README.md b/resources_servers/wmt_translation/README.md index 065116ceb2..f343e2b5e3 100644 --- a/resources_servers/wmt_translation/README.md +++ b/resources_servers/wmt_translation/README.md @@ -72,7 +72,7 @@ For an end-to-end SLURM run with COMET enabled, see the # Running servers (BLEU-only locally; flip compute_comet=true on cluster) gym env run \ --model-type vllm_model \ - --resource-server wmt_translation \ + --resources-server wmt_translation \ ++wmt_translation.resources_servers.wmt_translation.compute_comet=false # Collecting rollouts (5-example smoke test) diff --git a/resources_servers/workplace_assistant/README.md b/resources_servers/workplace_assistant/README.md index 0bc16882fb..6a9c21bd65 100644 --- a/resources_servers/workplace_assistant/README.md +++ b/resources_servers/workplace_assistant/README.md @@ -13,7 +13,7 @@ Spin up server: ``` gym env run \ --model-type openai_model \ - --resource-server workplace_assistant + --resources-server workplace_assistant ``` Collect trajectories: diff --git a/resources_servers/xlam_fc/README.md b/resources_servers/xlam_fc/README.md index 5de238d123..8079381a96 100644 --- a/resources_servers/xlam_fc/README.md +++ b/resources_servers/xlam_fc/README.md @@ -11,7 +11,7 @@ python resources_servers/xlam_fc/generate_dataset.py ```bash gym env run \ --model-type vllm_model \ - --resource-server xlam_fc + --resources-server xlam_fc ``` ```bash diff --git a/resources_servers/xstest/README.md b/resources_servers/xstest/README.md index 8be3b56e70..82d46c0b15 100644 --- a/resources_servers/xstest/README.md +++ b/resources_servers/xstest/README.md @@ -101,11 +101,11 @@ contrast_privacy ```bash # For chat completions endpoints (vLLM, NIM, etc.): gym env run \ - --resource-server xstest \ + --resources-server xstest \ --model-type vllm_model # For OpenAI Responses API endpoints: -# gym env run --resource-server xstest --model-type openai_model +# gym env run --resources-server xstest --model-type openai_model gym eval run --no-serve \ --agent xstest_simple_agent \ diff --git a/responses_api_agents/claude_code_agent/README.md b/responses_api_agents/claude_code_agent/README.md index e4b7ee41fb..0f61020d10 100644 --- a/responses_api_agents/claude_code_agent/README.md +++ b/responses_api_agents/claude_code_agent/README.md @@ -29,7 +29,7 @@ anthropic_base_url: http://localhost:8000 No model server is needed for basic eval. To extend this agent to training, a model server should be developed that handles messages endpoint. For evals with the current version, just pass the resources server config, which includes the agent server config, as is the current standard in NeMo Gym: ```bash -gym env run --resource-server reasoning_gym/reasoning_gym_claude_code_agent +gym env run --resources-server reasoning_gym/reasoning_gym_claude_code_agent ``` ### Run the agent diff --git a/responses_api_agents/hermes_agent/README.md b/responses_api_agents/hermes_agent/README.md index 6ba99677ba..a136108312 100644 --- a/responses_api_agents/hermes_agent/README.md +++ b/responses_api_agents/hermes_agent/README.md @@ -14,7 +14,7 @@ policy_model_name: gpt-4o ```bash gym env run \ - --resource-server math_with_judge/math_with_judge_hermes_agent \ + --resources-server math_with_judge/math_with_judge_hermes_agent \ --model-type openai_model ``` diff --git a/responses_api_agents/langgraph_agent/README.md b/responses_api_agents/langgraph_agent/README.md index 88b36d591b..c9a229b7da 100644 --- a/responses_api_agents/langgraph_agent/README.md +++ b/responses_api_agents/langgraph_agent/README.md @@ -10,7 +10,7 @@ Please note that agents such as parallel thinking which produce non-monotonicall ```bash gym env run \ - --resource-server reasoning_gym/reflection_agent \ + --resources-server reasoning_gym/reflection_agent \ --model-type vllm_model ``` diff --git a/responses_api_agents/mini_swe_agent/README.md b/responses_api_agents/mini_swe_agent/README.md index 57d5fa1d0f..823def449a 100644 --- a/responses_api_agents/mini_swe_agent/README.md +++ b/responses_api_agents/mini_swe_agent/README.md @@ -108,7 +108,7 @@ gym dataset download --storage gitlab \ # Start server gym env run \ - --resource-server mini_swe_agent \ + --resources-server mini_swe_agent \ --model-type openai_model & \ '+mini_swe_simple_agent.responses_api_agents.mini_swe_agent.cache_dir_template=/path/to/images/xingyaoww_sweb.eval.x86_64.\{instance_id\}.sif' \ +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.run_golden=False \ diff --git a/responses_api_models/azure_openai_model/README.md b/responses_api_models/azure_openai_model/README.md index 43563a8e2d..d0e9ffedda 100644 --- a/responses_api_models/azure_openai_model/README.md +++ b/responses_api_models/azure_openai_model/README.md @@ -17,7 +17,7 @@ Set the API version. It usually looks something like "2024-10-21". ```bash gym env run \ --model-type azure_openai_model \ - --resource-server equivalence_llm_judge \ + --resources-server equivalence_llm_judge \ +policy_model.responses_api_models.azure_openai_model.default_query.api-version= ``` diff --git a/responses_api_models/local_vllm_model/README.md b/responses_api_models/local_vllm_model/README.md index 6e78803523..78429dede7 100644 --- a/responses_api_models/local_vllm_model/README.md +++ b/responses_api_models/local_vllm_model/README.md @@ -6,7 +6,7 @@ Run this on a single GPU node! Set tensor_parallel_size * data_parallel_size to ```bash gym env run \ - --resource-server example_single_tool_call \ + --resources-server example_single_tool_call \ --config responses_api_models/local_vllm_model/configs/nano_v3_single_node.yaml &> temp.log & \ ++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.tensor_parallel_size=4 \ ++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.data_parallel_size=2 \ diff --git a/tests/unit_tests/test_benchmarks.py b/tests/unit_tests/test_benchmarks.py index 0858e3d88a..e0c5ece28f 100644 --- a/tests/unit_tests/test_benchmarks.py +++ b/tests/unit_tests/test_benchmarks.py @@ -92,7 +92,7 @@ def test_resolves_domain_and_terms_from_real_config(self) -> None: domain, terms = _benchmark_extras(bench) assert domain == "math" - # The resource server's inner name and the dataset name are recovered for search. + # The resources server's inner name and the dataset name are recovered for search. assert "math_with_judge" in terms assert "aime24" in terms @@ -134,7 +134,7 @@ def test_query_matches_domain(self, capsys) -> None: assert "aime24" not in out def test_query_matches_resource_server(self, capsys) -> None: - # "judge" only appears via aime24's resource server name. + # "judge" only appears via aime24's resources server name. out = self._run("judge", self._benchmarks(), capsys) assert "aime24" in out assert "gpqa_diamond" not in out diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 644ea060aa..99bb820b62 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -250,7 +250,7 @@ def test_no_resource_server_runs_all(self, monkeypatch: MonkeyPatch) -> None: assert overrides == [] def test_resource_server_name_translates_to_entrypoint(self, monkeypatch: MonkeyPatch) -> None: - target, overrides = _dispatch_for(monkeypatch, ["env", "test", "--resource-server", "gpqa"]) + target, overrides = _dispatch_for(monkeypatch, ["env", "test", "--resources-server", "gpqa"]) assert target == "nemo_gym.cli.env:test" assert overrides == ["+entrypoint=resources_servers/gpqa"] @@ -486,14 +486,14 @@ def test_short_alias_on_env_run(self, monkeypatch: MonkeyPatch) -> None: class TestEnvInitFlags: def test_resource_server_translates_to_entrypoint(self, monkeypatch: MonkeyPatch) -> None: - target, overrides = _dispatch_for(monkeypatch, ["env", "init", "--resource-server", "my_server"]) + target, overrides = _dispatch_for(monkeypatch, ["env", "init", "--resources-server", "my_server"]) assert target == "nemo_gym.cli.env:init_resources_server" assert overrides == ["+entrypoint=resources_servers/my_server"] class TestEnvPackagesFlags: def test_flags(self, monkeypatch: MonkeyPatch) -> None: - target, overrides = _dispatch_for(monkeypatch, ["env", "packages", "--resource-server", "gpqa", "--outdated"]) + target, overrides = _dispatch_for(monkeypatch, ["env", "packages", "--resources-server", "gpqa", "--outdated"]) assert target == "nemo_gym.cli.env:pip_list" assert set(overrides) == { "+entrypoint=resources_servers/gpqa", @@ -539,12 +539,12 @@ def test_version_json_dispatches_with_override(self, monkeypatch: MonkeyPatch) - def test_env_packages_json_maps_to_uv_format(self, monkeypatch: MonkeyPatch) -> None: # env packages delegates to `uv pip list`, so --json maps onto its own --format=json rather than +json=true. - target, overrides = _dispatch_for(monkeypatch, ["env", "packages", "--resource-server", "mcqa", "--json"]) + target, overrides = _dispatch_for(monkeypatch, ["env", "packages", "--resources-server", "mcqa", "--json"]) assert target == "nemo_gym.cli.env:pip_list" assert set(overrides) == {"+entrypoint=resources_servers/mcqa", "+format=json"} def test_env_packages_without_json(self, monkeypatch: MonkeyPatch) -> None: - _, overrides = _dispatch_for(monkeypatch, ["env", "packages", "--resource-server", "mcqa"]) + _, overrides = _dispatch_for(monkeypatch, ["env", "packages", "--resources-server", "mcqa"]) assert overrides == ["+entrypoint=resources_servers/mcqa"] @@ -592,7 +592,7 @@ def test_config_without_verbose_keeps_level(self, monkeypatch: MonkeyPatch) -> N class TestAssetSelectors: - """Named selectors (--benchmark, --resource-server, --model-type) that resolve a name to a default config path. + """Named selectors (--benchmark, --resources-server, --model-type) that resolve a name to a default config path. Each example mirrors a real invocation from the docs/READMEs, so the sugar stays faithful to the documented config paths it replaces. The legacy `+config_paths=[...]` form each one is derived from is cited inline. @@ -608,10 +608,10 @@ class TestAssetSelectors: # benchmarks/gpqa/README.md: ng_run "+config_paths=[benchmarks/gpqa/config.yaml]" (start a benchmark's servers) (["env", "run", "--benchmark", "gpqa"], "benchmarks/gpqa/config.yaml"), # README.md / quickstart.mdx: resources_servers/mcqa/configs/mcqa.yaml - (["env", "run", "--resource-server", "mcqa"], "resources_servers/mcqa/configs/mcqa.yaml"), + (["env", "run", "--resources-server", "mcqa"], "resources_servers/mcqa/configs/mcqa.yaml"), # model-server/vllm.mdx: resources_servers/example_multi_step/configs/example_multi_step.yaml ( - ["env", "run", "--resource-server", "example_multi_step"], + ["env", "run", "--resources-server", "example_multi_step"], "resources_servers/example_multi_step/configs/example_multi_step.yaml", ), # README.md / quickstart.mdx: responses_api_models/openai_model/configs/openai_model.yaml @@ -632,7 +632,7 @@ def test_quickstart_resource_server_plus_model(self, monkeypatch: MonkeyPatch) - # ng_run "+config_paths=[resources_servers/mcqa/configs/mcqa.yaml, # responses_api_models/openai_model/configs/openai_model.yaml]" target, overrides = _dispatch_for( - monkeypatch, ["env", "run", "--resource-server", "mcqa", "--model-type", "openai_model"] + monkeypatch, ["env", "run", "--resources-server", "mcqa", "--model-type", "openai_model"] ) assert target == "nemo_gym.cli.env:run" paths, others = _split_overrides(overrides) @@ -664,7 +664,7 @@ def test_cli_reference_e2e_rollout_example(self, monkeypatch: MonkeyPatch) -> No [ "eval", "run", - "--resource-server", + "--resources-server", "math_with_judge", "--model-type", "openai_model", @@ -694,7 +694,7 @@ def test_cli_reference_prepare_data_example(self, monkeypatch: MonkeyPatch) -> N [ "dataset", "collate", - "--resource-server", + "--resources-server", "example_multi_step", "--mode", "example_validation", @@ -713,7 +713,7 @@ def test_cli_reference_prepare_data_example(self, monkeypatch: MonkeyPatch) -> N def test_resource_server_flavor_syntax(self, monkeypatch: MonkeyPatch) -> None: # `/` picks a named config inside the server's configs/ dir; math_with_judge ships several # flavoured configs (see reference/faq.mdx, which pairs a math_with_judge dataset flavour for profiling). - _, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--resource-server", "math_with_judge/dapo17k"]) + _, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--resources-server", "math_with_judge/dapo17k"]) assert overrides == [ f"+config_paths=[{WORKING_DIR / 'resources_servers/math_with_judge/configs/dapo17k.yaml'}]" ] @@ -731,7 +731,7 @@ def test_selectors_merge_into_single_config_paths(self, monkeypatch: MonkeyPatch # _split_overrides asserts they coalesce into a single token. _, overrides = _dispatch_for( monkeypatch, - ["eval", "run", "--config", "extra.yaml", "--resource-server", "mcqa", "--model-type", "openai_model"], + ["eval", "run", "--config", "extra.yaml", "--resources-server", "mcqa", "--model-type", "openai_model"], ) paths, others = _split_overrides(overrides) assert paths == { @@ -754,7 +754,7 @@ def test_unknown_benchmark_errors_with_available_hint(self, monkeypatch: MonkeyP def test_unknown_flavor_error_points_at_configs_dir(self, monkeypatch: MonkeyPatch, capsys) -> None: # For a known server with an unknown flavor, the hint should point at that server's configs/ dir. monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) - monkeypatch.setattr(sys, "argv", ["gym", "env", "run", "--resource-server", "mcqa/nope"]) + monkeypatch.setattr(sys, "argv", ["gym", "env", "run", "--resources-server", "mcqa/nope"]) with pytest.raises(SystemExit): main() err = capsys.readouterr().err @@ -803,7 +803,7 @@ def test_misspelled_component_name(self, monkeypatch: MonkeyPatch, capsys) -> No def test_misspelled_component_flavor(self, monkeypatch: MonkeyPatch, capsys) -> None: err = self._run_expecting_exit( - monkeypatch, capsys, ["eval", "run", "--resource-server", "math_with_judge/dapo17"] + monkeypatch, capsys, ["eval", "run", "--resources-server", "math_with_judge/dapo17"] ) assert "Did you mean `dapo17k`?" in err From 68d5fb62d1b5c2d3c9baec8fdd91e25c3d133e28 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 24 Jun 2026 10:43:29 +0200 Subject: [PATCH 29/40] fix: add backward compatible imports for moved functions Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/benchmarks.py | 15 ++++++ nemo_gym/cli/__init__.py | 47 ++++++++++------ nemo_gym/cli/_compat.py | 56 +++++++++++++++++++ nemo_gym/dataset_orchestrator.py | 16 ++++++ nemo_gym/gitlab_utils.py | 14 +++++ nemo_gym/prompt.py | 13 +++++ nemo_gym/reward_profile.py | 13 +++++ nemo_gym/rollout_collection.py | 14 +++++ nemo_gym/train_data_utils.py | 13 +++++ tests/unit_tests/test_cli_compat.py | 84 +++++++++++++++++++++++++++++ 10 files changed, 268 insertions(+), 17 deletions(-) create mode 100644 nemo_gym/cli/_compat.py create mode 100644 tests/unit_tests/test_cli_compat.py diff --git a/nemo_gym/benchmarks.py b/nemo_gym/benchmarks.py index 7d57c6a2f6..5a598b62e2 100644 --- a/nemo_gym/benchmarks.py +++ b/nemo_gym/benchmarks.py @@ -98,3 +98,18 @@ def _load_benchmarks_from_config_paths(config_paths: List[Path]) -> Dict[str, Be benchmarks_dict[maybe_bc.name] = maybe_bc return benchmarks_dict + + +# Backward-compatibility shims (CLI refactor): these symbols moved to `nemo_gym.cli.eval`. +# Re-exported lazily to avoid a circular import; accessing them emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 + + +__getattr__ = moved_attr_getter( + __name__, + { + "list_benchmarks": "nemo_gym.cli.eval", + "PrepareBenchmarkConfig": "nemo_gym.cli.eval", + "prepare_benchmark": "nemo_gym.cli.eval", + }, +) diff --git a/nemo_gym/cli/__init__.py b/nemo_gym/cli/__init__.py index be1e5395bb..43f54bc668 100644 --- a/nemo_gym/cli/__init__.py +++ b/nemo_gym/cli/__init__.py @@ -13,22 +13,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any +from nemo_gym.cli._compat import moved_attr_getter -# NOTE: The cli module was re-structured but NeMo-RL relies on these two imports. -# They resolve lazily so importing this package (which happens on every `gym` invocation) -# doesn't eagerly pull in hydra, wandb and ray at startup. -_LEGACY_EXPORTS = { - "RunHelper": "nemo_gym.cli.env", - "GlobalConfigDictParserConfig": "nemo_gym.global_config", -} - - -def __getattr__(name: str) -> Any: - module_path = _LEGACY_EXPORTS.get(name) - if module_path is None: - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - import importlib - - return getattr(importlib.import_module(module_path), name) +# The single `nemo_gym/cli.py` module was split into the `nemo_gym.cli` package. Its public +# surface is re-exported lazily from the new submodules so existing `from nemo_gym.cli import X` +# imports (e.g. NeMo-RL's `RunHelper` / `GlobalConfigDictParserConfig`) keep working. Lazy +# resolution keeps `import nemo_gym.cli` cheap — it doesn't eagerly pull in hydra, wandb or ray — +# and avoids circular imports. Each access emits a DeprecationWarning pointing at the new path. +__getattr__ = moved_attr_getter( + __name__, + { + # Server orchestration + run/test entry points (old `nemo_gym/cli.py`) + "RunHelper": "nemo_gym.cli.env", + "RunConfig": "nemo_gym.cli.env", + "TestConfig": "nemo_gym.cli.env", + "TestAllConfig": "nemo_gym.cli.env", + "PipListConfig": "nemo_gym.cli.env", + "run": "nemo_gym.cli.env", + "test": "nemo_gym.cli.env", + "test_all": "nemo_gym.cli.env", + "init_resources_server": "nemo_gym.cli.env", + "dump_config": "nemo_gym.cli.env", + "status": "nemo_gym.cli.env", + "pip_list": "nemo_gym.cli.env", + "e2e_rollout_collection": "nemo_gym.cli.eval", + "dev_test": "nemo_gym.cli.dev", + "VersionConfig": "nemo_gym.cli.general", + "version": "nemo_gym.cli.general", + # Never actually lived in cli; reachable via the old module's top-level import. + "GlobalConfigDictParserConfig": "nemo_gym.global_config", + }, +) diff --git a/nemo_gym/cli/_compat.py b/nemo_gym/cli/_compat.py new file mode 100644 index 0000000000..f3c6934796 --- /dev/null +++ b/nemo_gym/cli/_compat.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Backward-compatibility helpers for symbols relocated during the CLI refactor. + +It provides a lazy import mechanism for symbols that were moved to the `nemo_gym.cli` +package and emits a `DeprecationWarning` pointing at the correct import path. + +Note that an eager re-export `from nemo_gym.cli.x import foo` at the top of a +core module would create a circular import. +""" + +import importlib +import warnings +from typing import Any, Callable, Mapping + + +def moved_attr_getter(module_name: str, moved: Mapping[str, str]) -> Callable[[str], Any]: + """Build a module `__getattr__` that re-exports relocated symbols with a deprecation warning. + + Args: + module_name: The `__name__` of the module installing the shim (the old location). + moved: Maps each old attribute name to its new location. The value is either + `"package.module"` (same attribute name) or `"package.module:new_name"` + when the symbol was also renamed. + + Returns: + A function suitable for assigning to a module's `__getattr__`. + """ + + def __getattr__(name: str) -> Any: + target = moved.get(name) + if target is None: + raise AttributeError(f"module {module_name!r} has no attribute {name!r}") + new_module, _, new_name = target.partition(":") + new_name = new_name or name + warnings.warn( + f"`{module_name}.{name}` has moved to `{new_module}.{new_name}` and will be removed in a " + f"future release. Update your import to `from {new_module} import {new_name}`.", + DeprecationWarning, + stacklevel=2, + ) + return getattr(importlib.import_module(new_module), new_name) + + return __getattr__ diff --git a/nemo_gym/dataset_orchestrator.py b/nemo_gym/dataset_orchestrator.py index a2cf9bdb14..baf6686c44 100644 --- a/nemo_gym/dataset_orchestrator.py +++ b/nemo_gym/dataset_orchestrator.py @@ -53,3 +53,19 @@ def upload_jsonl_dataset_to_hf_maybe_delete( if delete_from_gitlab: delete_jsonl_dataset_from_gitlab(gitlab_model_name) + + +# Backward-compatibility shims (CLI refactor): these CLI entry points moved to `nemo_gym.cli.dataset`. +# Re-exported lazily to avoid a circular import; accessing them emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 + + +__getattr__ = moved_attr_getter( + __name__, + { + "upload_jsonl_dataset_to_hf_cli": "nemo_gym.cli.dataset", + "upload_jsonl_dataset_to_hf_and_delete_gitlab_cli": "nemo_gym.cli.dataset", + "download_jsonl_dataset_from_hf_cli": "nemo_gym.cli.dataset", + "delete_jsonl_dataset_from_gitlab_cli": "nemo_gym.cli.dataset", + }, +) diff --git a/nemo_gym/gitlab_utils.py b/nemo_gym/gitlab_utils.py index e83b85787d..b96093e0bc 100644 --- a/nemo_gym/gitlab_utils.py +++ b/nemo_gym/gitlab_utils.py @@ -110,3 +110,17 @@ def is_model_in_gitlab(model_name: str) -> bool: # pragma: no cover def delete_model_from_gitlab(model_name: str) -> None: # pragma: no cover client = create_mlflow_client() client.delete_registered_model(model_name) + + +# Backward-compatibility shims (CLI refactor): these CLI entry points moved to `nemo_gym.cli.dataset`. +# Re-exported lazily to avoid a circular import; accessing them emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 + + +__getattr__ = moved_attr_getter( + __name__, + { + "upload_jsonl_dataset_cli": "nemo_gym.cli.dataset", + "download_jsonl_dataset_cli": "nemo_gym.cli.dataset", + }, +) diff --git a/nemo_gym/prompt.py b/nemo_gym/prompt.py index fae834d7ca..1fcb45990a 100644 --- a/nemo_gym/prompt.py +++ b/nemo_gym/prompt.py @@ -161,3 +161,16 @@ class MaterializePromptsConfig(BaseNeMoGymCLIConfig): input_jsonl_fpath: str = Field(description="Raw JSONL data (no responses_create_params.input).") prompt_config: str = Field(description="Path to prompt YAML file to apply.") output_jsonl_fpath: str = Field(description="Output path for materialized JSONL with populated prompts.") + + +# Backward-compatibility shim (CLI refactor): this CLI entry point moved to `nemo_gym.cli.dataset`. +# Re-exported lazily to avoid a circular import; accessing it emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 + + +__getattr__ = moved_attr_getter( + __name__, + { + "materialize_prompts_cli": "nemo_gym.cli.dataset", + }, +) diff --git a/nemo_gym/reward_profile.py b/nemo_gym/reward_profile.py index de395f4798..ba3f0e992e 100644 --- a/nemo_gym/reward_profile.py +++ b/nemo_gym/reward_profile.py @@ -700,3 +700,16 @@ def compute_aggregate_metrics( agent_metrics=serialized_agent, key_metrics=key_metrics, ) + + +# Backward-compatibility shim (CLI refactor): this CLI entry point moved to `nemo_gym.cli.eval`. +# Re-exported lazily to avoid a circular import; accessing it emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 + + +__getattr__ = moved_attr_getter( + __name__, + { + "reward_profile": "nemo_gym.cli.eval", + }, +) diff --git a/nemo_gym/rollout_collection.py b/nemo_gym/rollout_collection.py index ed53bf4904..72a0ac9378 100644 --- a/nemo_gym/rollout_collection.py +++ b/nemo_gym/rollout_collection.py @@ -711,3 +711,17 @@ async def run_from_config(self, config: RolloutAggregationConfig) -> Optional[Pa Aggregate metrics: {aggregate_metrics_fpath}""") return aggregate_metrics_fpath + + +# Backward-compatibility shims (CLI refactor): these CLI entry points moved to `nemo_gym.cli.eval`. +# Re-exported lazily to avoid a circular import; accessing them emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 + + +__getattr__ = moved_attr_getter( + __name__, + { + "collect_rollouts": "nemo_gym.cli.eval", + "aggregate_rollouts": "nemo_gym.cli.eval", + }, +) diff --git a/nemo_gym/train_data_utils.py b/nemo_gym/train_data_utils.py index 2ea6e4842d..56c33fecd9 100644 --- a/nemo_gym/train_data_utils.py +++ b/nemo_gym/train_data_utils.py @@ -822,3 +822,16 @@ def validate_backend_credentials(backend: str) -> tuple[bool, str]: ) return True, "" + + +# Backward-compatibility shim (CLI refactor): this CLI entry point moved to `nemo_gym.cli.dataset`. +# Re-exported lazily to avoid a circular import; accessing it emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 + + +__getattr__ = moved_attr_getter( + __name__, + { + "prepare_data": "nemo_gym.cli.dataset", + }, +) diff --git a/tests/unit_tests/test_cli_compat.py b/tests/unit_tests/test_cli_compat.py new file mode 100644 index 0000000000..85fb156976 --- /dev/null +++ b/tests/unit_tests/test_cli_compat.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The CLI refactor moved these symbols into `nemo_gym.cli`; the old import paths must keep +working (with a `DeprecationWarning`) for the deprecation period. Remove once they are dropped.""" + +import importlib + +import pytest + + +# (old_module, old_name, new_module, new_name) for each relocated symbol. +MOVED_SYMBOLS = [ + ("nemo_gym.benchmarks", "list_benchmarks", "nemo_gym.cli.eval", "list_benchmarks"), + ("nemo_gym.benchmarks", "PrepareBenchmarkConfig", "nemo_gym.cli.eval", "PrepareBenchmarkConfig"), + ("nemo_gym.benchmarks", "prepare_benchmark", "nemo_gym.cli.eval", "prepare_benchmark"), + ( + "nemo_gym.dataset_orchestrator", + "upload_jsonl_dataset_to_hf_cli", + "nemo_gym.cli.dataset", + "upload_jsonl_dataset_to_hf_cli", + ), + ( + "nemo_gym.dataset_orchestrator", + "upload_jsonl_dataset_to_hf_and_delete_gitlab_cli", + "nemo_gym.cli.dataset", + "upload_jsonl_dataset_to_hf_and_delete_gitlab_cli", + ), + ( + "nemo_gym.dataset_orchestrator", + "download_jsonl_dataset_from_hf_cli", + "nemo_gym.cli.dataset", + "download_jsonl_dataset_from_hf_cli", + ), + ( + "nemo_gym.dataset_orchestrator", + "delete_jsonl_dataset_from_gitlab_cli", + "nemo_gym.cli.dataset", + "delete_jsonl_dataset_from_gitlab_cli", + ), + ("nemo_gym.gitlab_utils", "upload_jsonl_dataset_cli", "nemo_gym.cli.dataset", "upload_jsonl_dataset_cli"), + ("nemo_gym.gitlab_utils", "download_jsonl_dataset_cli", "nemo_gym.cli.dataset", "download_jsonl_dataset_cli"), + ("nemo_gym.prompt", "materialize_prompts_cli", "nemo_gym.cli.dataset", "materialize_prompts_cli"), + ("nemo_gym.reward_profile", "reward_profile", "nemo_gym.cli.eval", "reward_profile"), + ("nemo_gym.rollout_collection", "collect_rollouts", "nemo_gym.cli.eval", "collect_rollouts"), + ("nemo_gym.rollout_collection", "aggregate_rollouts", "nemo_gym.cli.eval", "aggregate_rollouts"), + ("nemo_gym.train_data_utils", "prepare_data", "nemo_gym.cli.dataset", "prepare_data"), + ("nemo_gym.cli", "RunHelper", "nemo_gym.cli.env", "RunHelper"), + ("nemo_gym.cli", "RunConfig", "nemo_gym.cli.env", "RunConfig"), + ("nemo_gym.cli", "TestConfig", "nemo_gym.cli.env", "TestConfig"), + ("nemo_gym.cli", "TestAllConfig", "nemo_gym.cli.env", "TestAllConfig"), + ("nemo_gym.cli", "PipListConfig", "nemo_gym.cli.env", "PipListConfig"), + ("nemo_gym.cli", "run", "nemo_gym.cli.env", "run"), + ("nemo_gym.cli", "test", "nemo_gym.cli.env", "test"), + ("nemo_gym.cli", "test_all", "nemo_gym.cli.env", "test_all"), + ("nemo_gym.cli", "init_resources_server", "nemo_gym.cli.env", "init_resources_server"), + ("nemo_gym.cli", "dump_config", "nemo_gym.cli.env", "dump_config"), + ("nemo_gym.cli", "status", "nemo_gym.cli.env", "status"), + ("nemo_gym.cli", "pip_list", "nemo_gym.cli.env", "pip_list"), + ("nemo_gym.cli", "e2e_rollout_collection", "nemo_gym.cli.eval", "e2e_rollout_collection"), + ("nemo_gym.cli", "dev_test", "nemo_gym.cli.dev", "dev_test"), + ("nemo_gym.cli", "VersionConfig", "nemo_gym.cli.general", "VersionConfig"), + ("nemo_gym.cli", "version", "nemo_gym.cli.general", "version"), + ("nemo_gym.cli", "GlobalConfigDictParserConfig", "nemo_gym.global_config", "GlobalConfigDictParserConfig"), +] + + +@pytest.mark.parametrize("old_module, old_name, new_module, new_name", MOVED_SYMBOLS) +def test_deprecated_import_still_resolves_and_warns(old_module, old_name, new_module, new_name) -> None: + old_mod = importlib.import_module(old_module) + with pytest.warns(DeprecationWarning): + resolved = getattr(old_mod, old_name) + assert resolved is getattr(importlib.import_module(new_module), new_name) From e1f416e2a203db45f92dea15943aeaef389bf568 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 24 Jun 2026 10:45:06 +0200 Subject: [PATCH 30/40] chore: remove legacy display_help (is not used anywhere) Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/general.py | 29 ----------------------------- tests/unit_tests/test_cli.py | 17 ----------------- 2 files changed, 46 deletions(-) diff --git a/nemo_gym/cli/general.py b/nemo_gym/cli/general.py index 6aca20da0e..29d7152ccf 100644 --- a/nemo_gym/cli/general.py +++ b/nemo_gym/cli/general.py @@ -17,12 +17,10 @@ import os import platform import sys -from importlib.metadata import entry_points from importlib.metadata import version as md_version from subprocess import Popen import psutil -import rich from pydantic import Field from nemo_gym import PARENT_DIR, __version__ @@ -30,33 +28,6 @@ from nemo_gym.global_config import JSON_OUTPUT_KEY_NAME, get_global_config_dict -def display_help_legacy(): - """ - Display a list of available NeMo Gym CLI commands. - - Examples: - - ```bash - ng_help - ``` - """ - global_config_dict = get_global_config_dict() - # Just here for help - BaseNeMoGymCLIConfig.model_validate(global_config_dict) - - eps = entry_points().select(group="console_scripts") - project_scripts = {ep.name: ep.value for ep in eps if ep.name.startswith(("nemo_gym_", "ng_"))} - rich.print("""Run a command with `+h=true` or `+help=true` to see more detailed information! - -[bold]Available CLI scripts[/bold] ------------------""") - for script in project_scripts: - if not script.startswith("ng_"): - continue - - print(script) - - class VersionConfig(BaseNeMoGymCLIConfig): """ Display gym version and system information. diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index 654c847c8e..8e3d7edd7d 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -13,10 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import shutil -import sys import tomllib from importlib import import_module -from io import StringIO from pathlib import Path from subprocess import TimeoutExpired from unittest.mock import MagicMock, patch @@ -34,7 +32,6 @@ _select_shard, init_resources_server, ) -from nemo_gym.cli.general import display_help_legacy from nemo_gym.config_types import ResourcesServerInstanceConfig @@ -86,20 +83,6 @@ def test_pyproject_scripts_are_importable(self) -> None: target = getattr(import_module(module), fn) assert callable(target), f"{script_name} -> {import_path} is not callable" - def test_display_help_discovers_scripts(self) -> None: - with MonkeyPatch.context() as mp: - mp.setattr(nemo_gym.global_config, "_GLOBAL_CONFIG_DICT", OmegaConf.create({})) - - text_trap = StringIO() - mp.setattr(sys, "stdout", text_trap) - - display_help_legacy() - - output = text_trap.getvalue() - assert "ng_help" in output - assert "ng_run" in output - assert "ng_collect_rollouts" in output - def test_init_resources_server_includes_domain(self) -> None: """Test that init_resources_server creates a config with the required domain field.""" From ac9e79133a78da09697390775644b5e23423c16e Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 24 Jun 2026 11:25:57 +0200 Subject: [PATCH 31/40] chore: move cli_setup_command.py to cli submodule Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/env.py | 2 +- nemo_gym/cli/setup_command.py | 204 +++++++++++++++++++++ nemo_gym/cli_setup_command.py | 200 ++------------------ tests/unit_tests/test_cli_compat.py | 2 + tests/unit_tests/test_cli_setup_command.py | 16 +- 5 files changed, 228 insertions(+), 196 deletions(-) create mode 100644 nemo_gym/cli/setup_command.py diff --git a/nemo_gym/cli/env.py b/nemo_gym/cli/env.py index 7554c0f625..b81bcb59f5 100644 --- a/nemo_gym/cli/env.py +++ b/nemo_gym/cli/env.py @@ -37,7 +37,7 @@ from tqdm.auto import tqdm from nemo_gym import PARENT_DIR, ROOT_DIR -from nemo_gym.cli_setup_command import run_command, setup_env_command +from nemo_gym.cli.setup_command import run_command, setup_env_command from nemo_gym.config_types import BaseNeMoGymCLIConfig from nemo_gym.global_config import ( DRY_RUN_KEY_NAME, diff --git a/nemo_gym/cli/setup_command.py b/nemo_gym/cli/setup_command.py new file mode 100644 index 0000000000..3c979b07c5 --- /dev/null +++ b/nemo_gym/cli/setup_command.py @@ -0,0 +1,204 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import importlib.metadata +import os +from os import environ +from pathlib import Path +from subprocess import Popen +from sys import stderr, stdout + +from omegaconf import DictConfig + +from nemo_gym import PARENT_DIR +from nemo_gym.global_config import ( + HEAD_SERVER_DEPS_KEY_NAME, + NEMO_GYM_LOG_DIR_KEY_NAME, + PIP_INSTALL_VERBOSE_KEY_NAME, + PYTHON_VERSION_KEY_NAME, + SKIP_VENV_IF_PRESENT_KEY_NAME, + UV_CACHE_DIR_KEY_NAME, + UV_PIP_SET_PYTHON_KEY_NAME, + UV_VENV_DIR_KEY_NAME, + get_global_config_dict, +) + + +def _get_nemo_gym_install_flags() -> str: + """ + Build uv pip install flags for nemo-gym in sub-venvs. + + Supports: + - Pre-release versions via NEMO_GYM_ALLOW_PRERELEASE=true + - Custom PyPI indexes via UV_INDEX_URL, UV_EXTRA_INDEX_URL, UV_INDEX_STRATEGY + - Auto-detection of parent venv version for consistency + + Returns: + String of flags to add to 'uv pip install nemo-gym' + Example: "--pre --index-url https://test.pypi.org/simple/ ==0.2.1rc0" + """ + flags = "" + + # 1. Pre-release flag + allow_prerelease = os.getenv("NEMO_GYM_ALLOW_PRERELEASE", "").lower() == "true" + if allow_prerelease: + flags += "--pre " + # When pre-releases are enabled, also use unsafe-best-match strategy if not already set + if not os.getenv("UV_INDEX_STRATEGY"): + flags += "--index-strategy unsafe-best-match " + # Pin fastapi<1.0 to avoid broken test.pypi package + flags += "'fastapi<1.0' " + + # 2. Index URLs (respects uv's standard env vars) + index_url = os.getenv("UV_INDEX_URL") + if index_url: + flags += f"--index-url {index_url} " + + extra_index_url = os.getenv("UV_EXTRA_INDEX_URL") + if extra_index_url: + flags += f"--extra-index-url {extra_index_url} " + + # Explicit index strategy (overrides auto-set above) + index_strategy = os.getenv("UV_INDEX_STRATEGY") + if index_strategy: + flags += f"--index-strategy {index_strategy} " + + return flags + + +def _get_nemo_gym_version_spec(is_editable_install: bool) -> str: + """ + Detect nemo-gym version from parent venv and return version specifier. + + Args: + is_editable_install: Whether nemo-gym is installed in editable mode in parent venv + + Returns: + Version specifier string (e.g., "==0.2.1rc0") or empty string + """ + # Don't pin version for editable installs (development mode) + if is_editable_install: + return "" + + try: + parent_version = importlib.metadata.version("nemo-gym") + # Pin to exact version for consistency between parent and sub-venvs + return f"=={parent_version}" + except importlib.metadata.PackageNotFoundError: + # nemo-gym not installed in parent venv (shouldn't happen, but be safe) + return "" + + +def setup_env_command(dir_path: Path, global_config_dict: DictConfig, prefix: str) -> str: + head_server_deps = global_config_dict[HEAD_SERVER_DEPS_KEY_NAME] + + root_venv_path = global_config_dict[UV_VENV_DIR_KEY_NAME] + if Path(root_venv_path).resolve() != PARENT_DIR.resolve(): + venv_path = Path(root_venv_path, *dir_path.parts[-2:], ".venv").absolute() + else: + venv_path = (dir_path / ".venv").absolute() + + uv_venv_cmd = f"uv venv --seed --allow-existing --python {global_config_dict[PYTHON_VERSION_KEY_NAME]} {venv_path}" + + venv_python_fpath = venv_path / "bin/python" + venv_activate_fpath = venv_path / "bin/activate" + skip_venv_if_present = global_config_dict[SKIP_VENV_IF_PRESENT_KEY_NAME] + should_skip_venv_setup = bool(skip_venv_if_present) and venv_python_fpath.exists() and venv_activate_fpath.exists() + + # explicitly set python path if specified. In Google colab, ng_run fails due to uv pip install falls back to system python (/usr) without this and errors. + # not needed for most clusters. should be safe in all scenarios, but only minimally tested outside of colab. + # see discussion and examples here: https://github.com/NVIDIA-NeMo/Gym/pull/526#issuecomment-3676230383 + uv_pip_set_python = global_config_dict.get(UV_PIP_SET_PYTHON_KEY_NAME, False) + uv_pip_python_flag = f"--python {venv_python_fpath} " if uv_pip_set_python else "" + + verbose_flag = "-v " if global_config_dict.get(PIP_INSTALL_VERBOSE_KEY_NAME) else "" + + is_editable_install = (dir_path.resolve() / "../../pyproject.toml").exists() + + if should_skip_venv_setup: + env_setup_cmd = f"source {venv_activate_fpath}" + else: + has_pyproject_toml = (dir_path / "pyproject.toml").exists() + has_requirements_txt = (dir_path / "requirements.txt").exists() + if has_pyproject_toml and has_requirements_txt: + raise RuntimeError( + f"Found both pyproject.toml and requirements.txt for uv venv setup in server dir: {dir_path}. Please only use one or the other!" + ) + elif has_pyproject_toml: + if is_editable_install: + install_cmd = ( + f"""uv pip install {verbose_flag}{uv_pip_python_flag}'-e .' {" ".join(head_server_deps)}""" + ) + else: + # install nemo-gym from pypi instead of relative path in pyproject.toml + # with support for pre-releases, custom indexes, and version pinning + install_flags = _get_nemo_gym_install_flags() + version_spec = _get_nemo_gym_version_spec(is_editable_install) + install_cmd = ( + f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}nemo-gym{version_spec} && """ + f"""uv pip install {verbose_flag}{uv_pip_python_flag}--no-sources '-e .' {" ".join(head_server_deps)}""" + ) + elif has_requirements_txt: + if is_editable_install: + install_cmd = f"""uv pip install {verbose_flag}{uv_pip_python_flag}-r requirements.txt {" ".join(head_server_deps)}""" + else: + # install nemo-gym from pypi instead of relative path in requirements.txt + # with support for pre-releases, custom indexes, and version pinning + install_flags = _get_nemo_gym_install_flags() + version_spec = _get_nemo_gym_version_spec(is_editable_install) + install_cmd = ( + f"""(echo 'nemo-gym{version_spec}' && grep -v -F '../..' requirements.txt) | """ + f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}-r /dev/stdin {" ".join(head_server_deps)}""" + ) + else: + raise RuntimeError( + f"Missing pyproject.toml or requirements.txt for uv venv setup in server dir: {dir_path}" + ) + + prefix_cmd = f" > >(sed 's/^/({prefix}) /') 2> >(sed 's/^/({prefix}) /' >&2)" + env_setup_cmd = f"{uv_venv_cmd}{prefix_cmd} && source {venv_activate_fpath} && {install_cmd}{prefix_cmd}" + + return f"cd {dir_path} && {env_setup_cmd}" + + +def run_command(command: str, working_dir_path: Path, server_name: str = "") -> Popen: + global_config_dict = get_global_config_dict() + + work_dir = f"{working_dir_path.absolute()}" + custom_env = environ.copy() + py_path = custom_env.get("PYTHONPATH", None) + if py_path is not None: + custom_env["PYTHONPATH"] = f"{work_dir}:{py_path}" + else: + custom_env["PYTHONPATH"] = work_dir + + custom_env["UV_CACHE_DIR"] = global_config_dict[UV_CACHE_DIR_KEY_NAME] + + log_dir = global_config_dict.get(NEMO_GYM_LOG_DIR_KEY_NAME) + if log_dir: + safe_name = (server_name or working_dir_path.name).replace("/", "_") + log_path = Path(log_dir) / f"{safe_name}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + command = f"set -o pipefail; ({command}) 2>&1 | tee -a {log_path}" + + redirect_stdout = stdout + redirect_stderr = stderr + return Popen( + command, + executable="/bin/bash", + shell=True, + env=custom_env, + stdout=redirect_stdout, + stderr=redirect_stderr, + ) diff --git a/nemo_gym/cli_setup_command.py b/nemo_gym/cli_setup_command.py index 9e5a72181d..9d112ab251 100644 --- a/nemo_gym/cli_setup_command.py +++ b/nemo_gym/cli_setup_command.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,193 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import importlib.metadata -import os -from os import environ -from pathlib import Path -from subprocess import Popen -from sys import stderr, stdout +"""Backward-compatibility shim: this module moved to `nemo_gym.cli.setup_command`. -from omegaconf import DictConfig - -from nemo_gym import PARENT_DIR -from nemo_gym.global_config import ( - HEAD_SERVER_DEPS_KEY_NAME, - NEMO_GYM_LOG_DIR_KEY_NAME, - PIP_INSTALL_VERBOSE_KEY_NAME, - PYTHON_VERSION_KEY_NAME, - SKIP_VENV_IF_PRESENT_KEY_NAME, - UV_CACHE_DIR_KEY_NAME, - UV_PIP_SET_PYTHON_KEY_NAME, - UV_VENV_DIR_KEY_NAME, - get_global_config_dict, -) - - -def _get_nemo_gym_install_flags() -> str: - """ - Build uv pip install flags for nemo-gym in sub-venvs. - - Supports: - - Pre-release versions via NEMO_GYM_ALLOW_PRERELEASE=true - - Custom PyPI indexes via UV_INDEX_URL, UV_EXTRA_INDEX_URL, UV_INDEX_STRATEGY - - Auto-detection of parent venv version for consistency - - Returns: - String of flags to add to 'uv pip install nemo-gym' - Example: "--pre --index-url https://test.pypi.org/simple/ ==0.2.1rc0" - """ - flags = "" - - # 1. Pre-release flag - allow_prerelease = os.getenv("NEMO_GYM_ALLOW_PRERELEASE", "").lower() == "true" - if allow_prerelease: - flags += "--pre " - # When pre-releases are enabled, also use unsafe-best-match strategy if not already set - if not os.getenv("UV_INDEX_STRATEGY"): - flags += "--index-strategy unsafe-best-match " - # Pin fastapi<1.0 to avoid broken test.pypi package - flags += "'fastapi<1.0' " - - # 2. Index URLs (respects uv's standard env vars) - index_url = os.getenv("UV_INDEX_URL") - if index_url: - flags += f"--index-url {index_url} " - - extra_index_url = os.getenv("UV_EXTRA_INDEX_URL") - if extra_index_url: - flags += f"--extra-index-url {extra_index_url} " - - # Explicit index strategy (overrides auto-set above) - index_strategy = os.getenv("UV_INDEX_STRATEGY") - if index_strategy: - flags += f"--index-strategy {index_strategy} " - - return flags - - -def _get_nemo_gym_version_spec(is_editable_install: bool) -> str: - """ - Detect nemo-gym version from parent venv and return version specifier. - - Args: - is_editable_install: Whether nemo-gym is installed in editable mode in parent venv - - Returns: - Version specifier string (e.g., "==0.2.1rc0") or empty string - """ - # Don't pin version for editable installs (development mode) - if is_editable_install: - return "" - - try: - parent_version = importlib.metadata.version("nemo-gym") - # Pin to exact version for consistency between parent and sub-venvs - return f"=={parent_version}" - except importlib.metadata.PackageNotFoundError: - # nemo-gym not installed in parent venv (shouldn't happen, but be safe) - return "" +Accessing its public helpers here resolves them lazily from the new location with a +`DeprecationWarning`. Remove once the old import path is dropped. +""" +from nemo_gym.cli._compat import moved_attr_getter -def setup_env_command(dir_path: Path, global_config_dict: DictConfig, prefix: str) -> str: - head_server_deps = global_config_dict[HEAD_SERVER_DEPS_KEY_NAME] - root_venv_path = global_config_dict[UV_VENV_DIR_KEY_NAME] - if Path(root_venv_path).resolve() != PARENT_DIR.resolve(): - venv_path = Path(root_venv_path, *dir_path.parts[-2:], ".venv").absolute() - else: - venv_path = (dir_path / ".venv").absolute() - - uv_venv_cmd = f"uv venv --seed --allow-existing --python {global_config_dict[PYTHON_VERSION_KEY_NAME]} {venv_path}" - - venv_python_fpath = venv_path / "bin/python" - venv_activate_fpath = venv_path / "bin/activate" - skip_venv_if_present = global_config_dict[SKIP_VENV_IF_PRESENT_KEY_NAME] - should_skip_venv_setup = bool(skip_venv_if_present) and venv_python_fpath.exists() and venv_activate_fpath.exists() - - # explicitly set python path if specified. In Google colab, ng_run fails due to uv pip install falls back to system python (/usr) without this and errors. - # not needed for most clusters. should be safe in all scenarios, but only minimally tested outside of colab. - # see discussion and examples here: https://github.com/NVIDIA-NeMo/Gym/pull/526#issuecomment-3676230383 - uv_pip_set_python = global_config_dict.get(UV_PIP_SET_PYTHON_KEY_NAME, False) - uv_pip_python_flag = f"--python {venv_python_fpath} " if uv_pip_set_python else "" - - verbose_flag = "-v " if global_config_dict.get(PIP_INSTALL_VERBOSE_KEY_NAME) else "" - - is_editable_install = (dir_path.resolve() / "../../pyproject.toml").exists() - - if should_skip_venv_setup: - env_setup_cmd = f"source {venv_activate_fpath}" - else: - has_pyproject_toml = (dir_path / "pyproject.toml").exists() - has_requirements_txt = (dir_path / "requirements.txt").exists() - if has_pyproject_toml and has_requirements_txt: - raise RuntimeError( - f"Found both pyproject.toml and requirements.txt for uv venv setup in server dir: {dir_path}. Please only use one or the other!" - ) - elif has_pyproject_toml: - if is_editable_install: - install_cmd = ( - f"""uv pip install {verbose_flag}{uv_pip_python_flag}'-e .' {" ".join(head_server_deps)}""" - ) - else: - # install nemo-gym from pypi instead of relative path in pyproject.toml - # with support for pre-releases, custom indexes, and version pinning - install_flags = _get_nemo_gym_install_flags() - version_spec = _get_nemo_gym_version_spec(is_editable_install) - install_cmd = ( - f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}nemo-gym{version_spec} && """ - f"""uv pip install {verbose_flag}{uv_pip_python_flag}--no-sources '-e .' {" ".join(head_server_deps)}""" - ) - elif has_requirements_txt: - if is_editable_install: - install_cmd = f"""uv pip install {verbose_flag}{uv_pip_python_flag}-r requirements.txt {" ".join(head_server_deps)}""" - else: - # install nemo-gym from pypi instead of relative path in requirements.txt - # with support for pre-releases, custom indexes, and version pinning - install_flags = _get_nemo_gym_install_flags() - version_spec = _get_nemo_gym_version_spec(is_editable_install) - install_cmd = ( - f"""(echo 'nemo-gym{version_spec}' && grep -v -F '../..' requirements.txt) | """ - f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}-r /dev/stdin {" ".join(head_server_deps)}""" - ) - else: - raise RuntimeError( - f"Missing pyproject.toml or requirements.txt for uv venv setup in server dir: {dir_path}" - ) - - prefix_cmd = f" > >(sed 's/^/({prefix}) /') 2> >(sed 's/^/({prefix}) /' >&2)" - env_setup_cmd = f"{uv_venv_cmd}{prefix_cmd} && source {venv_activate_fpath} && {install_cmd}{prefix_cmd}" - - return f"cd {dir_path} && {env_setup_cmd}" - - -def run_command(command: str, working_dir_path: Path, server_name: str = "") -> Popen: - global_config_dict = get_global_config_dict() - - work_dir = f"{working_dir_path.absolute()}" - custom_env = environ.copy() - py_path = custom_env.get("PYTHONPATH", None) - if py_path is not None: - custom_env["PYTHONPATH"] = f"{work_dir}:{py_path}" - else: - custom_env["PYTHONPATH"] = work_dir - - custom_env["UV_CACHE_DIR"] = global_config_dict[UV_CACHE_DIR_KEY_NAME] - - log_dir = global_config_dict.get(NEMO_GYM_LOG_DIR_KEY_NAME) - if log_dir: - safe_name = (server_name or working_dir_path.name).replace("/", "_") - log_path = Path(log_dir) / f"{safe_name}.log" - log_path.parent.mkdir(parents=True, exist_ok=True) - command = f"set -o pipefail; ({command}) 2>&1 | tee -a {log_path}" - - redirect_stdout = stdout - redirect_stderr = stderr - return Popen( - command, - executable="/bin/bash", - shell=True, - env=custom_env, - stdout=redirect_stdout, - stderr=redirect_stderr, - ) +__getattr__ = moved_attr_getter( + __name__, + { + "run_command": "nemo_gym.cli.setup_command", + "setup_env_command": "nemo_gym.cli.setup_command", + }, +) diff --git a/tests/unit_tests/test_cli_compat.py b/tests/unit_tests/test_cli_compat.py index 85fb156976..758d662c5b 100644 --- a/tests/unit_tests/test_cli_compat.py +++ b/tests/unit_tests/test_cli_compat.py @@ -73,6 +73,8 @@ ("nemo_gym.cli", "VersionConfig", "nemo_gym.cli.general", "VersionConfig"), ("nemo_gym.cli", "version", "nemo_gym.cli.general", "version"), ("nemo_gym.cli", "GlobalConfigDictParserConfig", "nemo_gym.global_config", "GlobalConfigDictParserConfig"), + ("nemo_gym.cli_setup_command", "run_command", "nemo_gym.cli.setup_command", "run_command"), + ("nemo_gym.cli_setup_command", "setup_env_command", "nemo_gym.cli.setup_command", "setup_env_command"), ] diff --git a/tests/unit_tests/test_cli_setup_command.py b/tests/unit_tests/test_cli_setup_command.py index b751c4472d..4f531eedc1 100644 --- a/tests/unit_tests/test_cli_setup_command.py +++ b/tests/unit_tests/test_cli_setup_command.py @@ -19,8 +19,8 @@ import pytest from pytest import MonkeyPatch, raises -import nemo_gym.cli_setup_command -from nemo_gym.cli_setup_command import ( +import nemo_gym.cli.setup_command +from nemo_gym.cli.setup_command import ( _get_nemo_gym_install_flags, _get_nemo_gym_version_spec, run_command, @@ -243,15 +243,15 @@ def test_uv_venv_dir_and_skip_install_when_venv_present(self, tmp_path: Path) -> class TestCLISetupCommandRunCommand: def _setup(self, monkeypatch: MonkeyPatch) -> tuple[MagicMock, MagicMock]: Popen_mock = MagicMock() - monkeypatch.setattr(nemo_gym.cli_setup_command, "Popen", Popen_mock) + monkeypatch.setattr(nemo_gym.cli.setup_command, "Popen", Popen_mock) get_global_config_dict_mock = MagicMock(return_value={"uv_cache_dir": "default uv cache dir"}) - monkeypatch.setattr(nemo_gym.cli_setup_command, "get_global_config_dict", get_global_config_dict_mock) + monkeypatch.setattr(nemo_gym.cli.setup_command, "get_global_config_dict", get_global_config_dict_mock) - monkeypatch.setattr(nemo_gym.cli_setup_command, "environ", dict()) + monkeypatch.setattr(nemo_gym.cli.setup_command, "environ", dict()) - monkeypatch.setattr(nemo_gym.cli_setup_command, "stdout", "stdout") - monkeypatch.setattr(nemo_gym.cli_setup_command, "stderr", "stderr") + monkeypatch.setattr(nemo_gym.cli.setup_command, "stdout", "stdout") + monkeypatch.setattr(nemo_gym.cli.setup_command, "stderr", "stderr") return Popen_mock, get_global_config_dict_mock @@ -276,7 +276,7 @@ def test_sanity(self, monkeypatch: MonkeyPatch) -> None: def test_custom_pythonpath(self, monkeypatch: MonkeyPatch) -> None: Popen_mock, get_global_config_dict_mock = self._setup(monkeypatch) - monkeypatch.setattr(nemo_gym.cli_setup_command, "environ", {"PYTHONPATH": "existing pythonpath"}) + monkeypatch.setattr(nemo_gym.cli.setup_command, "environ", {"PYTHONPATH": "existing pythonpath"}) run_command( command="my command", From 46a8f0e81b8013926e1189164144790e5eb7785e Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 24 Jun 2026 11:53:20 +0200 Subject: [PATCH 32/40] feat: add --environment flag and update commands in READMEs for envs Signed-off-by: Marta Stepniewska-Dziubinska --- environments/abstention/README.md | 16 ++++++++-------- environments/arc_agi/README.md | 6 +++--- environments/blackjack/README.md | 10 +++++----- environments/calendar/README.md | 14 ++++++-------- environments/calendar_v2/README.md | 14 ++++++-------- environments/circle_click/README.md | 4 ++-- environments/circle_count/README.md | 4 ++-- environments/code_gen/README.md | 13 +++++-------- environments/ether0/README.md | 10 +++++----- environments/instruction_following/README.md | 10 +++++----- environments/reasoning_gym/README.md | 12 ++++++------ environments/workplace_assistant/README.md | 2 +- nemo_gym/cli/main.py | 15 ++++++++++++++- tests/unit_tests/test_cli_main.py | 6 ++++++ 14 files changed, 74 insertions(+), 62 deletions(-) diff --git a/environments/abstention/README.md b/environments/abstention/README.md index cd68ed8973..7809edc87a 100644 --- a/environments/abstention/README.md +++ b/environments/abstention/README.md @@ -15,20 +15,20 @@ The dataset used is [HotPotQA](https://hotpotqa.github.io/) (fullwiki split). ## Running servers ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -environments/abstention/config.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env run \ + --environment abstention \ + --model-type openai_model \ +abstention.resources_servers.abstention.judge_model_server.name=policy_model ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=abstention_simple_agent \ - +input_jsonl_fpath=environments/abstention/data/example.jsonl \ - +output_jsonl_fpath=results/abstention_verify_responses.jsonl \ - +limit=3 +gym eval run --no-serve \ + --agent abstention_simple_agent \ + --input environments/abstention/data/example.jsonl \ + --output results/abstention_verify_responses.jsonl \ + --limit 3 ``` ## Preprocessing HotPotQA data diff --git a/environments/arc_agi/README.md b/environments/arc_agi/README.md index 86eac0ef72..7fbe827092 100644 --- a/environments/arc_agi/README.md +++ b/environments/arc_agi/README.md @@ -51,7 +51,7 @@ uv sync ### Start ARC-AGI environment (we can reuse the same one for ARC-AGI-1 and 2): ```bash -ng_run "+config_paths=[environments/arc_agi/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run --environment arc_agi --model-type vllm_model ``` @@ -59,12 +59,12 @@ ng_run "+config_paths=[environments/arc_agi/config.yaml,responses_api_models/vll ARC-AGI-1 example rollouts: ```bash -ng_collect_rollouts +agent_name=arc_agi_simple_agent +input_jsonl_fpath=environments/arc_agi/data/example_1.jsonl +output_jsonl_fpath=environments/arc_agi/data/example_1_rollouts.jsonl +gym eval run --no-serve --agent arc_agi_simple_agent --input environments/arc_agi/data/example_1.jsonl --output environments/arc_agi/data/example_1_rollouts.jsonl ``` ARC-AGI-2 example rollouts: ```bash -ng_collect_rollouts +agent_name=arc_agi_simple_agent +input_jsonl_fpath=environments/arc_agi/data/example_2.jsonl +output_jsonl_fpath=environments/arc_agi/data/example_2_rollouts.jsonl +gym eval run --no-serve --agent arc_agi_simple_agent --input environments/arc_agi/data/example_2.jsonl --output environments/arc_agi/data/example_2_rollouts.jsonl ``` For training, see the [docs](https://docs.nvidia.com/nemo/gym/latest/training-tutorials/nemo-rl-grpo/index.html). \ No newline at end of file diff --git a/environments/blackjack/README.md b/environments/blackjack/README.md index adfecbb546..2ef35c7628 100644 --- a/environments/blackjack/README.md +++ b/environments/blackjack/README.md @@ -9,7 +9,7 @@ Example data provided in `data/example.jsonl` (system prompt only, no verifier_m ## Run ```bash -ng_run "+config_paths=[environments/blackjack/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run --environment blackjack --model-type vllm_model ``` ## Data @@ -19,10 +19,10 @@ Each game is generated on the fly during `reset()`, so every row in `example.jso ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=blackjack_gymnasium_agent \ - +input_jsonl_fpath=environments/blackjack/data/example.jsonl \ - +output_jsonl_fpath=results/blackjack_rollouts.jsonl +gym eval run --no-serve \ + --agent blackjack_gymnasium_agent \ + --input environments/blackjack/data/example.jsonl \ + --output results/blackjack_rollouts.jsonl ``` diff --git a/environments/calendar/README.md b/environments/calendar/README.md index de05ef056a..f16a5db21c 100644 --- a/environments/calendar/README.md +++ b/environments/calendar/README.md @@ -16,9 +16,7 @@ Create an `env.yaml` file in the Gym root directory to specifying `policy_base_u The following is an example command for running this resources server along with an OpenAI model: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -environments/calendar/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run --environment calendar --model-type openai_model ``` ## Collecting rollouts @@ -26,11 +24,11 @@ ng_run "+config_paths=[$config_paths]" Rollouts can be collected using the example dataset as follows: ```bash -ng_collect_rollouts \ - +agent_name=calendar_simple_agent \ - +input_jsonl_fpath=environments/calendar/data/example.jsonl \ - +output_jsonl_fpath=results/example_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent calendar_simple_agent \ + --input environments/calendar/data/example.jsonl \ + --output results/example_rollouts.jsonl \ + --limit 5 ``` The input JSONL file should contain entries with: diff --git a/environments/calendar_v2/README.md b/environments/calendar_v2/README.md index 0e9138f9f8..4e80000792 100644 --- a/environments/calendar_v2/README.md +++ b/environments/calendar_v2/README.md @@ -16,9 +16,7 @@ The conversations in the dataset are generated using personas from the [nvidia/N The following is an example command for running this resources server along with an OpenAI model: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -environments/calendar_v2/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run --environment calendar_v2 --model-type openai_model ``` ## Collecting rollouts @@ -26,11 +24,11 @@ Create an `env.yaml` file in the Gym root directory to specifying `policy_base_u Rollouts can be collected using the example dataset as follows: ```bash -ng_collect_rollouts \ - +agent_name=calendar_simple_agent \ - +input_jsonl_fpath=data/example.jsonl \ - +output_jsonl_fpath=environments/calendar_v2/data/example.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent calendar_simple_agent \ + --input data/example.jsonl \ + --output environments/calendar_v2/data/example.jsonl \ + --limit 5 ``` The input JSONL file should contain entries with: diff --git a/environments/circle_click/README.md b/environments/circle_click/README.md index f83297a676..a748601093 100644 --- a/environments/circle_click/README.md +++ b/environments/circle_click/README.md @@ -12,8 +12,8 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & -ng_run "+config_paths=[environments/circle_click/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" & -ng_collect_rollouts +agent_name=circle_click_simple_agent +input_jsonl_fpath=environments/circle_click/data/example.jsonl +output_jsonl_fpath=environments/circle_click/data/example_rollouts.jsonl +limit=1 +gym env run --environment circle_click --model-type vllm_model & +gym eval run --no-serve --agent circle_click_simple_agent --input environments/circle_click/data/example.jsonl --output environments/circle_click/data/example_rollouts.jsonl --limit 1 ``` # Generating Data diff --git a/environments/circle_count/README.md b/environments/circle_count/README.md index bc7de17d45..2af1986cd8 100644 --- a/environments/circle_count/README.md +++ b/environments/circle_count/README.md @@ -19,8 +19,8 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & -ng_run "+config_paths=[environments/circle_count/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" & -ng_collect_rollouts +agent_name=circle_count_simple_agent +input_jsonl_fpath=environments/circle_count/data/example.jsonl +output_jsonl_fpath=environments/circle_count/data/example_rollouts.jsonl +limit=1 +gym env run --environment circle_count --model-type vllm_model & +gym eval run --no-serve --agent circle_count_simple_agent --input environments/circle_count/data/example.jsonl --output environments/circle_count/data/example_rollouts.jsonl --limit 1 ``` # Generating Data diff --git a/environments/code_gen/README.md b/environments/code_gen/README.md index 1b10d07744..3e5edf9b58 100644 --- a/environments/code_gen/README.md +++ b/environments/code_gen/README.md @@ -21,18 +21,15 @@ The dataset is in preparation and the example data can be found in `data/example We use the LiveCodeBench execution code. ### Example of rollouts and usage -Create an `env.yaml` file in the Gym root directory to specify the model endpoint and credentials. See [documentation](https://docs.nvidia.com/nemo/gym/reference/configuration#local-configuration-envyaml) for details. +Create an `env.yaml` file in the Gym root directory to specify the model endpoint and credentials. See [documentation](https://docs.nvidia.com/nemo/gym/reference/configuration#local-configuration-envyaml) for details. ```bash # Running the server -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -environments/code_gen/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env run --environment code_gen --model-type openai_model # Collect rollouts from example problems -ng_collect_rollouts +agent_name=code_gen_simple_agent \ - +input_jsonl_fpath=environments/code_gen/data/example.jsonl \ - +output_jsonl_fpath=environments/code_gen/data/example_rollouts.jsonl \ - +limit=null +gym eval run --no-serve --agent code_gen_simple_agent \ + --input environments/code_gen/data/example.jsonl \ + --output environments/code_gen/data/example_rollouts.jsonl ``` ## Licensing information diff --git a/environments/ether0/README.md b/environments/ether0/README.md index 1a77748666..d10191bf04 100644 --- a/environments/ether0/README.md +++ b/environments/ether0/README.md @@ -27,13 +27,13 @@ Start servers and collect rollouts ```bash # start vllm and nemo gym servers vllm serve futurehouse/ether0 & -ng_run "+config_paths=[environments/ether0/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" & +gym env run --environment ether0 --model-type vllm_model & # wait for above to be ready -ng_collect_rollouts \ - +agent_name=ether0_simple_agent \ - +input_jsonl_fpath=environments/ether0/data/example.jsonl \ - +output_jsonl_fpath=environments/ether0/data/ether0_rollouts.jsonl +gym eval run --no-serve \ + --agent ether0_simple_agent \ + --input environments/ether0/data/example.jsonl \ + --output environments/ether0/data/ether0_rollouts.jsonl tail -n 1 environments/ether0/data/ether0_rollouts.jsonl | jq | less ``` diff --git a/environments/instruction_following/README.md b/environments/instruction_following/README.md index 965ecbb67f..97c20184b5 100644 --- a/environments/instruction_following/README.md +++ b/environments/instruction_following/README.md @@ -47,14 +47,14 @@ The dataset can be found at https://huggingface.co/datasets/nvidia/Nemotron-RL-i ### Usage Create an `env.yaml` file in the Gym root directory to specifying `policy_base_url`, `policy_model_name`, and `policy_api_key`. See [documentation](https://docs.nvidia.com/nemo/gym/reference/configuration#local-configuration-envyaml) for details. ```bash -ng_run "+config_paths=[environments/instruction_following/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run --environment instruction_following --model-type vllm_model python environments/instruction_following/prepare.py -ng_collect_rollouts \ - +agent_name=instruction_following_simple_agent \ - +input_jsonl_fpath=environments/instruction_following/data/example.jsonl \ - +output_jsonl_fpath=results/instruction_following_rollouts.jsonl +limit=5 +gym eval run --no-serve \ + --agent instruction_following_simple_agent \ + --input environments/instruction_following/data/example.jsonl \ + --output results/instruction_following_rollouts.jsonl --limit 5 ``` ### Rollout example diff --git a/environments/reasoning_gym/README.md b/environments/reasoning_gym/README.md index c3f866f14f..f096069777 100644 --- a/environments/reasoning_gym/README.md +++ b/environments/reasoning_gym/README.md @@ -87,15 +87,15 @@ policy_model_name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 ## Launch nemo gym servers ```bash -ng_run "+config_paths=[environments/reasoning_gym/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env run --environment reasoning_gym --model-type vllm_model ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=reasoning_gym_simple_agent \ - +input_jsonl_fpath=environments/reasoning_gym/data/example.jsonl \ - +output_jsonl_fpath=results/reasoning_gym_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent reasoning_gym_simple_agent \ + --input environments/reasoning_gym/data/example.jsonl \ + --output results/reasoning_gym_rollouts.jsonl \ + --limit 5 ``` diff --git a/environments/workplace_assistant/README.md b/environments/workplace_assistant/README.md index d16862a74c..e21fc076a2 100644 --- a/environments/workplace_assistant/README.md +++ b/environments/workplace_assistant/README.md @@ -13,7 +13,7 @@ Spin up server: ``` gym env run \ --model-type openai_model \ - --config environments/workplace_assistant/config.yaml + --environment workplace_assistant ``` Collect trajectories: diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 29b8786f50..a2d425f113 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -150,6 +150,7 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: # resolving to `//[/].yaml`. A None default flavor falls back to the server name. _ASSETS = { "benchmark": ("benchmarks", "", "config"), + "environment": ("environments", "", "config"), "resources-server": ("resources_servers", "configs", None), "model-type": ("responses_api_models", "configs", None), } @@ -223,6 +224,7 @@ def _asset_selector(flag: str) -> Flag: BENCHMARK = _asset_selector("benchmark") +ENVIRONMENT = _asset_selector("environment") RESOURCES_SERVER_CONFIG = _asset_selector("resources-server") MODEL_TYPE = _asset_selector("model-type") @@ -408,7 +410,17 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: "env run": Command( target="nemo_gym.cli.env:run", summary="Start the servers.", - flags=(CONFIG, BENCHMARK, RESOURCES_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, MODEL, MODEL_URL, MODEL_API_KEY), + flags=( + CONFIG, + BENCHMARK, + ENVIRONMENT, + RESOURCES_SERVER_CONFIG, + MODEL_TYPE, + SEARCH_DIR, + MODEL, + MODEL_URL, + MODEL_API_KEY, + ), ), "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status.", flags=(JSON,)), "eval prepare": Command( @@ -422,6 +434,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: flags=( CONFIG, BENCHMARK, + ENVIRONMENT, RESOURCES_SERVER_CONFIG, MODEL_TYPE, SEARCH_DIR, diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 99bb820b62..69659155e1 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -726,6 +726,12 @@ def test_benchmark_flavor_syntax(self, monkeypatch: MonkeyPatch) -> None: ) assert overrides == [f"+config_paths=[{WORKING_DIR / 'benchmarks/finance_sec_search/config_web_search.yaml'}]"] + def test_environment_selector_resolves_to_config(self, monkeypatch: MonkeyPatch) -> None: + # Environments mirror benchmarks: `--environment ` loads `environments//config.yaml`. + # Used by `gym env run` / `gym eval run` for environments/ (see environments/*/README.md). + _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--environment", "circle_count"]) + assert overrides == [f"+config_paths=[{WORKING_DIR / 'environments/circle_count/config.yaml'}]"] + def test_selectors_merge_into_single_config_paths(self, monkeypatch: MonkeyPatch) -> None: # --config and multiple asset selectors all feed one +config_paths list (Hydra rejects duplicates). # _split_overrides asserts they coalesce into a single token. From 993bf7e06922c5fa2b393686f1292fa479484272 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 24 Jun 2026 13:41:31 +0200 Subject: [PATCH 33/40] fix: name gym env run -> gym env start Signed-off-by: Marta Stepniewska-Dziubinska --- .claude/skills/add-benchmark/SKILL.md | 2 +- .../references/error-profiles.md | 4 +-- .../references/request-boundary-visibility.md | 2 +- .../skills/nemo-gym-reward-profiling/SKILL.md | 6 ++-- .../references/quick-start.md | 2 +- .../references/error-profiles.md | 4 +-- .../references/request-boundary-visibility.md | 2 +- .../skills/nemo-gym-reward-profiling/SKILL.md | 6 ++-- .../references/quick-start.md | 2 +- CLAUDE.md | 2 +- README.md | 2 +- benchmarks/aime24-x/README.md | 2 +- benchmarks/aime25-x/README.md | 2 +- benchmarks/aime26/README.md | 2 +- benchmarks/answer-judge/README.md | 2 +- benchmarks/apex_shortlist/README.md | 2 +- benchmarks/arena_hard/README.md | 2 +- benchmarks/arena_hard_v2/README.md | 2 +- benchmarks/asr_leaderboard/README.md | 2 +- benchmarks/bigcodebench/README.md | 2 +- benchmarks/birdbench/README.md | 2 +- benchmarks/finance_sec_search/README.md | 2 +- benchmarks/flores200/README.md | 2 +- benchmarks/frontierscience_olympiad/README.md | 2 +- benchmarks/global-piqa/README.md | 2 +- benchmarks/gpqa-x/README.md | 2 +- benchmarks/gpqa/README.md | 2 +- benchmarks/graphwalks/README.md | 2 +- benchmarks/gsm8k/README.md | 2 +- benchmarks/hendrycks_math/README.md | 2 +- benchmarks/hle/README.md | 2 +- benchmarks/hmmt_feb25/README.md | 2 +- benchmarks/hmmt_nov25/README.md | 2 +- benchmarks/hotpotqa_closedbook/README.md | 2 +- benchmarks/human_eval/README.md | 4 +-- benchmarks/human_eval_infilling/README.md | 4 +-- benchmarks/ifeval/README.md | 2 +- benchmarks/imo_answerbench/README.md | 2 +- benchmarks/imo_gradingbench/README.md | 2 +- benchmarks/imo_proofbench/README.md | 2 +- benchmarks/ioi/README.md | 2 +- benchmarks/labbench2_vlm/README.md | 4 +-- benchmarks/librispeech_pc/README.md | 2 +- benchmarks/livecodebench-x/README.md | 4 +-- benchmarks/longbench_v2/README.md | 2 +- benchmarks/longcodebench/README.md | 2 +- benchmarks/m_arena_hard/README.md | 2 +- benchmarks/m_arena_hard_v2/README.md | 2 +- benchmarks/math-500/README.md | 2 +- benchmarks/mbpp/README.md | 4 +-- benchmarks/minif2f/README.md | 2 +- benchmarks/mmlu-redux/README.md | 2 +- benchmarks/mmlu/README.md | 2 +- benchmarks/mmlu_pro/README.md | 2 +- benchmarks/mmlu_prox/README.md | 2 +- benchmarks/mmmlu/README.md | 2 +- benchmarks/mobench/README.md | 2 +- benchmarks/mrcr/README.md | 2 +- benchmarks/musan/README.md | 2 +- benchmarks/numb3rs/README.md | 2 +- benchmarks/physics/README.md | 2 +- benchmarks/polymath/README.md | 2 +- benchmarks/proof-arena-judge/README.md | 2 +- benchmarks/proof_bench_judge/README.md | 2 +- benchmarks/proofnet/README.md | 2 +- benchmarks/putnam_bench/README.md | 2 +- benchmarks/simpleqa/README.md | 2 +- benchmarks/speed-bench/README.md | 2 +- benchmarks/supergpqa/README.md | 2 +- benchmarks/ugphysics/README.md | 2 +- benchmarks/wmt24pp/README.md | 2 +- environments/abstention/README.md | 2 +- environments/arc_agi/README.md | 2 +- environments/blackjack/README.md | 2 +- environments/calendar/README.md | 2 +- environments/calendar_v2/README.md | 2 +- environments/circle_click/README.md | 2 +- environments/circle_count/README.md | 2 +- environments/code_gen/README.md | 2 +- environments/ether0/README.md | 2 +- environments/instruction_following/README.md | 2 +- environments/reasoning_gym/README.md | 2 +- environments/workplace_assistant/README.md | 2 +- .../environments/new-environment.mdx | 2 +- .../single-step-environment.mdx | 6 ++-- .../verification-patterns/llm-as-judge.mdx | 6 ++-- .../latest/pages/get-started/quickstart.mdx | 4 +-- .../engineering-notes/system-design.mdx | 8 ++--- .../latest/pages/model-server/vllm.mdx | 4 +-- .../latest/pages/reference/cli-commands.mdx | 16 ++++----- .../latest/pages/reference/configuration.mdx | 12 +++---- fern/versions/latest/pages/reference/faq.mdx | 10 +++--- .../multi-environment-training.mdx | 6 ++-- .../pages/troubleshooting/configuration.mdx | 8 ++--- nemo_gym/cli/legacy.py | 2 +- nemo_gym/cli/main.py | 4 +-- nemo_gym/server_status.py | 2 +- resources_servers/abstention/README.md | 2 +- resources_servers/arc_agi/README.md | 2 +- resources_servers/arena_judge/README.md | 2 +- resources_servers/asr_with_pc/README.md | 2 +- resources_servers/aviary/README.md | 6 ++-- resources_servers/bigcodebench/README.md | 2 +- resources_servers/bird_sql/README.md | 2 +- resources_servers/blackjack/README.md | 2 +- .../browsecomp_advanced_harness/README.md | 2 +- resources_servers/calendar/README.md | 2 +- resources_servers/circle_click/README.md | 2 +- resources_servers/circle_count/README.md | 2 +- resources_servers/code_fim/README.md | 2 +- resources_servers/code_gen/README.md | 2 +- .../competitive_coding_challenges/README.md | 2 +- resources_servers/cvdp/README.md | 2 +- .../equivalence_llm_judge/README.md | 2 +- resources_servers/ether0/README.md | 2 +- resources_servers/evalplus/README.md | 2 +- .../example_multi_turn_gymnasium/README.md | 2 +- .../finance_sec_search/README.md | 6 ++-- .../format_verification/README.md | 4 +-- .../frontierscience_judge/README.md | 2 +- resources_servers/google_search/README.md | 2 +- resources_servers/gpqa_diamond/README.md | 4 +-- resources_servers/graphwalks/README.md | 2 +- resources_servers/grl_sokoban/README.md | 2 +- resources_servers/grl_tetris/README.md | 2 +- resources_servers/hotpotqa_qa/README.md | 2 +- resources_servers/ifbench/README.md | 2 +- resources_servers/imo_gradingbench/README.md | 2 +- .../imo_proofbench_judge/README.md | 2 +- .../instruction_following/README.md | 2 +- resources_servers/inverse_if/README.md | 4 +-- .../jailbreak_detection/README.md | 2 +- resources_servers/labbench2_vlm/README.md | 4 +-- resources_servers/longmt_eval/README.md | 2 +- .../math_advanced_calculations/README.md | 2 +- .../math_proof_judgement/README.md | 2 +- .../math_with_autograder/README.md | 2 +- resources_servers/math_with_code/README.md | 2 +- resources_servers/math_with_judge/README.md | 2 +- resources_servers/mcqa/README.md | 2 +- resources_servers/mrcr/README.md | 2 +- resources_servers/multichallenge/README.md | 4 +-- resources_servers/newton_bench/README.md | 2 +- resources_servers/ns_tools/README.md | 2 +- resources_servers/openenv/README.md | 10 +++--- resources_servers/physics_judge/README.md | 2 +- resources_servers/polymath/README.md | 2 +- resources_servers/rdkit_chemistry/README.md | 2 +- resources_servers/reasoning_gym/README.md | 2 +- resources_servers/simpleqa/README.md | 2 +- .../README.md | 2 +- resources_servers/speed_bench/README.md | 2 +- resources_servers/structeval/README.md | 2 +- .../structured_outputs/README.md | 4 +-- resources_servers/swerl_gen/README.md | 2 +- resources_servers/swerl_llm_judge/README.md | 2 +- resources_servers/tavily_search/README.md | 2 +- resources_servers/terminus_judge/README.md | 2 +- .../terminus_judge/scripts/README.md | 2 +- resources_servers/text_to_sql/README.md | 2 +- resources_servers/ugphysics_judge/README.md | 2 +- resources_servers/verifif/README.md | 4 +-- resources_servers/vlm_eval_kit/README.md | 2 +- resources_servers/wmt_translation/README.md | 2 +- .../workplace_assistant/README.md | 2 +- resources_servers/xlam_fc/README.md | 2 +- resources_servers/xstest/README.md | 4 +-- .../claude_code_agent/README.md | 2 +- responses_api_agents/harbor_agent/README.md | 6 ++-- responses_api_agents/hermes_agent/README.md | 2 +- .../langgraph_agent/README.md | 2 +- responses_api_agents/mini_swe_agent/README.md | 2 +- responses_api_agents/stirrup_agent/README.md | 2 +- responses_api_agents/swe_agents/README.md | 2 +- responses_api_agents/tau2/README.md | 2 +- .../verifiers_agent/README.md | 6 ++-- .../azure_openai_model/README.md | 2 +- .../local_vllm_model/README.md | 2 +- .../local_vllm_model_proxy/README.md | 2 +- tests/unit_tests/test_cli_main.py | 33 ++++++++++--------- tests/unit_tests/test_server_status.py | 2 +- 181 files changed, 260 insertions(+), 257 deletions(-) diff --git a/.claude/skills/add-benchmark/SKILL.md b/.claude/skills/add-benchmark/SKILL.md index 8b2c027831..eaa4334bdb 100644 --- a/.claude/skills/add-benchmark/SKILL.md +++ b/.claude/skills/add-benchmark/SKILL.md @@ -178,7 +178,7 @@ Test coverage must be >= 95%. Write tests for: verify pass, verify fail (wrong o ```bash # Start servers -gym env run \ +gym env start \ --config resources_servers/my_benchmark/configs/my_benchmark.yaml \ --model-type openai_model diff --git a/.claude/skills/nemo-gym-debugging/references/error-profiles.md b/.claude/skills/nemo-gym-debugging/references/error-profiles.md index 7c9d540f65..9705582aec 100644 --- a/.claude/skills/nemo-gym-debugging/references/error-profiles.md +++ b/.claude/skills/nemo-gym-debugging/references/error-profiles.md @@ -113,7 +113,7 @@ Next actions: Symptoms: -- `gym env run` or `gym eval run --no-serve` fails before launching servers +- `gym env start` or `gym eval run --no-serve` fails before launching servers - errors mention missing config keys, unknown overrides, or interpolation failures - wrong agent/resource server starts despite expected data @@ -134,7 +134,7 @@ Evidence to collect: Next actions: -- reduce to a minimal `gym env run --config ...` command +- reduce to a minimal `gym env start --config ...` command - add overrides back one at a time - trust target checkout code over copied command templates diff --git a/.claude/skills/nemo-gym-debugging/references/request-boundary-visibility.md b/.claude/skills/nemo-gym-debugging/references/request-boundary-visibility.md index 0b83647324..a56b2f1ee9 100644 --- a/.claude/skills/nemo-gym-debugging/references/request-boundary-visibility.md +++ b/.claude/skills/nemo-gym-debugging/references/request-boundary-visibility.md @@ -9,7 +9,7 @@ This is an escalation ladder. Do not patch agents, resource servers, model adapt 1. Enable Gym's existing request debug flag: ```bash - gym env run --config ... ++global_aiohttp_client_request_debug=True + gym env start --config ... ++global_aiohttp_client_request_debug=True ``` 2. Re-run the smallest failing case and capture stdout/stderr. diff --git a/.claude/skills/nemo-gym-reward-profiling/SKILL.md b/.claude/skills/nemo-gym-reward-profiling/SKILL.md index 20b42c7dab..ebcb7758db 100644 --- a/.claude/skills/nemo-gym-reward-profiling/SKILL.md +++ b/.claude/skills/nemo-gym-reward-profiling/SKILL.md @@ -2,7 +2,7 @@ name: nemo-gym-reward-profiling description: >- Use to help users get started with Nemo Gym reward profiling. Covers the basic - gym env run, gym eval run, and gym eval profile workflow, repeated rollouts, + gym env start, gym eval run, and gym eval profile workflow, repeated rollouts, materialized inputs, rollout JSONL artifacts, task and rollout identity, output inspection, partial profiling, and rollout_infos. For failed jobs, prefer nemo-gym-debugging. @@ -14,14 +14,14 @@ description: >- Use this skill when the user wants to run, understand, or lightly modify Nemo Gym reward profiling. Keep the answer oriented around the normal workflow: -`gym env run` starts model/resources servers, `gym eval run --no-serve` writes rollout artifacts, and `gym eval profile` generates profiling output from those artifacts. +`gym env start` starts model/resources servers, `gym eval run --no-serve` writes rollout artifacts, and `gym eval profile` generates profiling output from those artifacts. If the user is primarily debugging a failed job or stack trace, use the `nemo-gym-debugging` skill first. ## Basic Workflow 1. Identify the environment config paths and input JSONL. -2. Start Gym servers with `gym env run`. +2. Start Gym servers with `gym env start`. 3. Collect rollouts with `gym eval run --no-serve`; this writes `rollouts.jsonl` and `*_materialized_inputs.jsonl`. 4. Run `gym eval profile` on the materialized inputs and rollout JSONL to generate `*_reward_profiling.jsonl`. 5. Inspect line counts and profile rows. diff --git a/.claude/skills/nemo-gym-reward-profiling/references/quick-start.md b/.claude/skills/nemo-gym-reward-profiling/references/quick-start.md index 9626f30605..3b19d2e832 100644 --- a/.claude/skills/nemo-gym-reward-profiling/references/quick-start.md +++ b/.claude/skills/nemo-gym-reward-profiling/references/quick-start.md @@ -17,7 +17,7 @@ AGENT_NAME="your_agent_name" NUM_REPEATS=2 NUM_SAMPLES_IN_PARALLEL=8 -gym env run \ +gym env start \ --config your_model_config_paths \ --config your_env_config_paths \ --model "$POLICY_MODEL_NAME" \ diff --git a/.codex/skills/nemo-gym-debugging/references/error-profiles.md b/.codex/skills/nemo-gym-debugging/references/error-profiles.md index 7c9d540f65..9705582aec 100644 --- a/.codex/skills/nemo-gym-debugging/references/error-profiles.md +++ b/.codex/skills/nemo-gym-debugging/references/error-profiles.md @@ -113,7 +113,7 @@ Next actions: Symptoms: -- `gym env run` or `gym eval run --no-serve` fails before launching servers +- `gym env start` or `gym eval run --no-serve` fails before launching servers - errors mention missing config keys, unknown overrides, or interpolation failures - wrong agent/resource server starts despite expected data @@ -134,7 +134,7 @@ Evidence to collect: Next actions: -- reduce to a minimal `gym env run --config ...` command +- reduce to a minimal `gym env start --config ...` command - add overrides back one at a time - trust target checkout code over copied command templates diff --git a/.codex/skills/nemo-gym-debugging/references/request-boundary-visibility.md b/.codex/skills/nemo-gym-debugging/references/request-boundary-visibility.md index 0b83647324..a56b2f1ee9 100644 --- a/.codex/skills/nemo-gym-debugging/references/request-boundary-visibility.md +++ b/.codex/skills/nemo-gym-debugging/references/request-boundary-visibility.md @@ -9,7 +9,7 @@ This is an escalation ladder. Do not patch agents, resource servers, model adapt 1. Enable Gym's existing request debug flag: ```bash - gym env run --config ... ++global_aiohttp_client_request_debug=True + gym env start --config ... ++global_aiohttp_client_request_debug=True ``` 2. Re-run the smallest failing case and capture stdout/stderr. diff --git a/.codex/skills/nemo-gym-reward-profiling/SKILL.md b/.codex/skills/nemo-gym-reward-profiling/SKILL.md index 20b42c7dab..ebcb7758db 100644 --- a/.codex/skills/nemo-gym-reward-profiling/SKILL.md +++ b/.codex/skills/nemo-gym-reward-profiling/SKILL.md @@ -2,7 +2,7 @@ name: nemo-gym-reward-profiling description: >- Use to help users get started with Nemo Gym reward profiling. Covers the basic - gym env run, gym eval run, and gym eval profile workflow, repeated rollouts, + gym env start, gym eval run, and gym eval profile workflow, repeated rollouts, materialized inputs, rollout JSONL artifacts, task and rollout identity, output inspection, partial profiling, and rollout_infos. For failed jobs, prefer nemo-gym-debugging. @@ -14,14 +14,14 @@ description: >- Use this skill when the user wants to run, understand, or lightly modify Nemo Gym reward profiling. Keep the answer oriented around the normal workflow: -`gym env run` starts model/resources servers, `gym eval run --no-serve` writes rollout artifacts, and `gym eval profile` generates profiling output from those artifacts. +`gym env start` starts model/resources servers, `gym eval run --no-serve` writes rollout artifacts, and `gym eval profile` generates profiling output from those artifacts. If the user is primarily debugging a failed job or stack trace, use the `nemo-gym-debugging` skill first. ## Basic Workflow 1. Identify the environment config paths and input JSONL. -2. Start Gym servers with `gym env run`. +2. Start Gym servers with `gym env start`. 3. Collect rollouts with `gym eval run --no-serve`; this writes `rollouts.jsonl` and `*_materialized_inputs.jsonl`. 4. Run `gym eval profile` on the materialized inputs and rollout JSONL to generate `*_reward_profiling.jsonl`. 5. Inspect line counts and profile rows. diff --git a/.codex/skills/nemo-gym-reward-profiling/references/quick-start.md b/.codex/skills/nemo-gym-reward-profiling/references/quick-start.md index 9626f30605..3b19d2e832 100644 --- a/.codex/skills/nemo-gym-reward-profiling/references/quick-start.md +++ b/.codex/skills/nemo-gym-reward-profiling/references/quick-start.md @@ -17,7 +17,7 @@ AGENT_NAME="your_agent_name" NUM_REPEATS=2 NUM_SAMPLES_IN_PARALLEL=8 -gym env run \ +gym env start \ --config your_model_config_paths \ --config your_env_config_paths \ --model "$POLICY_MODEL_NAME" \ diff --git a/CLAUDE.md b/CLAUDE.md index 39d3edbfd6..2372b4fc86 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,7 +82,7 @@ uv venv && uv sync --extra dev --group docs pre-commit install # Run servers -gym env run \ +gym env start \ --resources-server example_single_tool_call \ --model-type vllm_model diff --git a/README.md b/README.md index 22c1de02bb..431628f73d 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Run your agent on a set of tasks and score the results. This example uses a simp NeMo Gym uses local servers to coordinate your model, agent, and task verification. Start them first: ```bash -gym env run \ +gym env start \ --resources-server mcqa \ --model-type openai_model ``` diff --git a/benchmarks/aime24-x/README.md b/benchmarks/aime24-x/README.md index 776fc0e17b..7fbe707469 100644 --- a/benchmarks/aime24-x/README.md +++ b/benchmarks/aime24-x/README.md @@ -36,7 +36,7 @@ python benchmarks/aime24-x/prepare.py --prompt_language en ## Quickstart ```bash -gym env run \ +gym env start \ --benchmark aime24-x \ --model-type vllm_model ``` diff --git a/benchmarks/aime25-x/README.md b/benchmarks/aime25-x/README.md index 6ae97da3cd..23f145180d 100644 --- a/benchmarks/aime25-x/README.md +++ b/benchmarks/aime25-x/README.md @@ -36,7 +36,7 @@ python benchmarks/aime25-x/prepare.py --prompt_language en ## Quickstart ```bash -gym env run \ +gym env start \ --benchmark aime25-x \ --model-type vllm_model ``` diff --git a/benchmarks/aime26/README.md b/benchmarks/aime26/README.md index 279c3c4f1a..8013220481 100644 --- a/benchmarks/aime26/README.md +++ b/benchmarks/aime26/README.md @@ -11,7 +11,7 @@ gym eval prepare --benchmark aime26 ## Run servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark aime26 ``` diff --git a/benchmarks/answer-judge/README.md b/benchmarks/answer-judge/README.md index ffd6fa066b..6418783221 100644 --- a/benchmarks/answer-judge/README.md +++ b/benchmarks/answer-judge/README.md @@ -16,7 +16,7 @@ here is the same deterministic `Judgement: Yes/No` parsing used by Skills' gym eval prepare --benchmark answer-judge # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark answer-judge diff --git a/benchmarks/apex_shortlist/README.md b/benchmarks/apex_shortlist/README.md index 526f69c7fa..bf41621ec1 100644 --- a/benchmarks/apex_shortlist/README.md +++ b/benchmarks/apex_shortlist/README.md @@ -35,7 +35,7 @@ Writes `data/apex_shortlist_benchmark.jsonl` with one row per problem: ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark apex_shortlist ``` diff --git a/benchmarks/arena_hard/README.md b/benchmarks/arena_hard/README.md index ab334cd230..d6038ce450 100644 --- a/benchmarks/arena_hard/README.md +++ b/benchmarks/arena_hard/README.md @@ -31,7 +31,7 @@ to pick the standard judge prompt. gym eval prepare --benchmark arena_hard # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark arena_hard diff --git a/benchmarks/arena_hard_v2/README.md b/benchmarks/arena_hard_v2/README.md index 31a4d551fa..5a8b160a25 100644 --- a/benchmarks/arena_hard_v2/README.md +++ b/benchmarks/arena_hard_v2/README.md @@ -34,7 +34,7 @@ repo, joins by `uid`, and emits one row per question with `question`, gym eval prepare --benchmark arena_hard_v2 # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark arena_hard_v2 diff --git a/benchmarks/asr_leaderboard/README.md b/benchmarks/asr_leaderboard/README.md index 0d1883fa0c..ce872ff0e9 100644 --- a/benchmarks/asr_leaderboard/README.md +++ b/benchmarks/asr_leaderboard/README.md @@ -32,7 +32,7 @@ Downloads the 8 ESB subsets (~tens of GB of FLAC) and writes ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark asr_leaderboard ``` diff --git a/benchmarks/bigcodebench/README.md b/benchmarks/bigcodebench/README.md index 16360c05bf..b638f3a0db 100644 --- a/benchmarks/bigcodebench/README.md +++ b/benchmarks/bigcodebench/README.md @@ -16,7 +16,7 @@ the `full` split (~1140 problems) is `bigcode/bigcodebench@v0.1.4`. gym eval prepare --benchmark bigcodebench # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark bigcodebench diff --git a/benchmarks/birdbench/README.md b/benchmarks/birdbench/README.md index f6e198e003..872193bb28 100644 --- a/benchmarks/birdbench/README.md +++ b/benchmarks/birdbench/README.md @@ -22,7 +22,7 @@ and writes `data/birdbench_benchmark.jsonl`. Each row has ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark birdbench ``` diff --git a/benchmarks/finance_sec_search/README.md b/benchmarks/finance_sec_search/README.md index deb3a4c675..28b6a2c7dc 100644 --- a/benchmarks/finance_sec_search/README.md +++ b/benchmarks/finance_sec_search/README.md @@ -47,7 +47,7 @@ JSONL to `data/`. ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark finance_sec_search/config_no_web_search ``` diff --git a/benchmarks/flores200/README.md b/benchmarks/flores200/README.md index 9b272195ef..df3ef4872b 100644 --- a/benchmarks/flores200/README.md +++ b/benchmarks/flores200/README.md @@ -41,7 +41,7 @@ only advertised on multi-node SLURM deployments via NeMo-Skills' override and rely on corpus-BLEU only: ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark flores200 \ "++flores200_wmt_translation_resources_server.resources_servers.wmt_translation.compute_comet=false" diff --git a/benchmarks/frontierscience_olympiad/README.md b/benchmarks/frontierscience_olympiad/README.md index e7c631df5e..da7e141641 100644 --- a/benchmarks/frontierscience_olympiad/README.md +++ b/benchmarks/frontierscience_olympiad/README.md @@ -39,7 +39,7 @@ example, the original Skills configuration uses `o3-mini-2025-01-31` via gym eval prepare --benchmark frontierscience_olympiad # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark frontierscience_olympiad diff --git a/benchmarks/global-piqa/README.md b/benchmarks/global-piqa/README.md index b3048b92a6..110434521e 100644 --- a/benchmarks/global-piqa/README.md +++ b/benchmarks/global-piqa/README.md @@ -19,7 +19,7 @@ server. gym eval prepare --benchmark global-piqa # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark global-piqa diff --git a/benchmarks/gpqa-x/README.md b/benchmarks/gpqa-x/README.md index 6043283d44..e17e2db729 100644 --- a/benchmarks/gpqa-x/README.md +++ b/benchmarks/gpqa-x/README.md @@ -37,7 +37,7 @@ python benchmarks/gpqa-x/prepare.py --prompt_language en ## Quickstart ```bash -gym env run \ +gym env start \ --benchmark gpqa-x \ --model-type vllm_model ``` diff --git a/benchmarks/gpqa/README.md b/benchmarks/gpqa/README.md index 679e9efb06..49f61c8073 100644 --- a/benchmarks/gpqa/README.md +++ b/benchmarks/gpqa/README.md @@ -16,7 +16,7 @@ This benchmark uses the `mcqa` resource server with the `mcqa_simple_agent`. gym eval prepare --benchmark gpqa # Start servers -gym env run \ +gym env start \ --benchmark gpqa \ --model-type vllm_model diff --git a/benchmarks/graphwalks/README.md b/benchmarks/graphwalks/README.md index e486653735..271864987d 100644 --- a/benchmarks/graphwalks/README.md +++ b/benchmarks/graphwalks/README.md @@ -47,7 +47,7 @@ python benchmarks/graphwalks/prepare.py \ ## Start environment ```bash -gym env run \ +gym env start \ --benchmark graphwalks \ --model-type vllm_model ``` diff --git a/benchmarks/gsm8k/README.md b/benchmarks/gsm8k/README.md index ef4b9d917c..4043b38e85 100644 --- a/benchmarks/gsm8k/README.md +++ b/benchmarks/gsm8k/README.md @@ -16,7 +16,7 @@ the expected answer is integer-valued), then renames `problem` -> gym eval prepare --benchmark gsm8k # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark gsm8k diff --git a/benchmarks/hendrycks_math/README.md b/benchmarks/hendrycks_math/README.md index d10253c03a..ffee07f070 100644 --- a/benchmarks/hendrycks_math/README.md +++ b/benchmarks/hendrycks_math/README.md @@ -15,7 +15,7 @@ applies Skills' renames (`answer` -> `expected_answer`, `question` -> gym eval prepare --benchmark hendrycks_math # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark hendrycks_math diff --git a/benchmarks/hle/README.md b/benchmarks/hle/README.md index b08d6486ef..ad4cc5fef4 100644 --- a/benchmarks/hle/README.md +++ b/benchmarks/hle/README.md @@ -34,7 +34,7 @@ Downloads `cais/hle`, filters to text-only questions, and writes ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark hle ``` diff --git a/benchmarks/hmmt_feb25/README.md b/benchmarks/hmmt_feb25/README.md index c890c0e390..a0806e2af1 100644 --- a/benchmarks/hmmt_feb25/README.md +++ b/benchmarks/hmmt_feb25/README.md @@ -37,7 +37,7 @@ Start the benchmark's servers (inherits `math_with_judge` in symbolic-only mode plus a vLLM model server — adjust the model config to match your deployment): ``` -gym env run \ +gym env start \ --benchmark hmmt_feb25 \ --model-type vllm_model ``` diff --git a/benchmarks/hmmt_nov25/README.md b/benchmarks/hmmt_nov25/README.md index 98a4ab4b85..f1764247c5 100644 --- a/benchmarks/hmmt_nov25/README.md +++ b/benchmarks/hmmt_nov25/README.md @@ -38,7 +38,7 @@ on the same inputs. gym eval prepare --benchmark hmmt_nov25 # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark hmmt_nov25 diff --git a/benchmarks/hotpotqa_closedbook/README.md b/benchmarks/hotpotqa_closedbook/README.md index 1c5bb2166b..5c93080f9d 100644 --- a/benchmarks/hotpotqa_closedbook/README.md +++ b/benchmarks/hotpotqa_closedbook/README.md @@ -25,7 +25,7 @@ gym eval prepare --benchmark hotpotqa_closedbook ## Run servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark hotpotqa_closedbook ``` diff --git a/benchmarks/human_eval/README.md b/benchmarks/human_eval/README.md index 32436d3d90..36898e2826 100644 --- a/benchmarks/human_eval/README.md +++ b/benchmarks/human_eval/README.md @@ -23,14 +23,14 @@ holds the dataset definition + prompt + prepare script. gym eval prepare --benchmark human_eval # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark human_eval # Collecting rollouts. The +prompt_config= override is required because # the prepared JSONL stores raw `question` rows (no `responses_create_params.input`); # gym eval run --no-serve does not pick up the `prompt_config:` field on the dataset -# entry in config.yaml the way gym env run does. +# entry in config.yaml the way gym env start does. gym eval run --no-serve \ --agent human_eval_evalplus_simple_agent \ --input benchmarks/human_eval/data/human_eval_benchmark.jsonl \ diff --git a/benchmarks/human_eval_infilling/README.md b/benchmarks/human_eval_infilling/README.md index 76aa13f800..be02a182e7 100644 --- a/benchmarks/human_eval_infilling/README.md +++ b/benchmarks/human_eval_infilling/README.md @@ -40,14 +40,14 @@ holds only the dataset definition + prompt + prepare script. gym eval prepare --benchmark human_eval_infilling # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark human_eval_infilling # Collecting rollouts. The +prompt_config= override is required because # the prepared JSONL stores raw rows (no `responses_create_params.input`); # gym eval run --no-serve does not pick up the `prompt_config:` field on the -# dataset entry in config.yaml the way gym env run does. +# dataset entry in config.yaml the way gym env start does. gym eval run --no-serve \ --agent human_eval_infilling_simple_agent \ --input benchmarks/human_eval_infilling/data/random_span.jsonl \ diff --git a/benchmarks/ifeval/README.md b/benchmarks/ifeval/README.md index 59035bb84b..a7c1626056 100644 --- a/benchmarks/ifeval/README.md +++ b/benchmarks/ifeval/README.md @@ -13,7 +13,7 @@ gym eval prepare --benchmark ifeval ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark ifeval ``` diff --git a/benchmarks/imo_answerbench/README.md b/benchmarks/imo_answerbench/README.md index 8fb41f3624..b1df651d5a 100644 --- a/benchmarks/imo_answerbench/README.md +++ b/benchmarks/imo_answerbench/README.md @@ -13,7 +13,7 @@ gym eval prepare --benchmark imo_answerbench ## Run servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark imo_answerbench ``` diff --git a/benchmarks/imo_gradingbench/README.md b/benchmarks/imo_gradingbench/README.md index dfb4235b2b..3be264572a 100644 --- a/benchmarks/imo_gradingbench/README.md +++ b/benchmarks/imo_gradingbench/README.md @@ -36,7 +36,7 @@ details. gym eval prepare --benchmark imo_gradingbench # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark imo_gradingbench diff --git a/benchmarks/imo_proofbench/README.md b/benchmarks/imo_proofbench/README.md index c67c634b6a..b12bc18d96 100644 --- a/benchmarks/imo_proofbench/README.md +++ b/benchmarks/imo_proofbench/README.md @@ -13,7 +13,7 @@ gym eval prepare --benchmark imo_proofbench ## Run servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark imo_proofbench \ +judge_base_url=https://generativelanguage.googleapis.com/v1beta/openai \ diff --git a/benchmarks/ioi/README.md b/benchmarks/ioi/README.md index b4d5ff4540..3fe5680a00 100644 --- a/benchmarks/ioi/README.md +++ b/benchmarks/ioi/README.md @@ -51,7 +51,7 @@ gym dataset collate --config benchmarks/ioi/config.yaml \ --output-dir benchmarks/ioi/data \ --mode benchmark_preparation -gym env run \ +gym env start \ --benchmark ioi \ --model-type vllm_model diff --git a/benchmarks/labbench2_vlm/README.md b/benchmarks/labbench2_vlm/README.md index a432ba413c..ebb17b365b 100644 --- a/benchmarks/labbench2_vlm/README.md +++ b/benchmarks/labbench2_vlm/README.md @@ -100,7 +100,7 @@ overwrite source data. The full config chain is required because ```bash # Start servers -gym env run \ +gym env start \ --benchmark labbench2_vlm \ --model-type openai_model @@ -130,7 +130,7 @@ PDFs as pages like other PDF tasks. ### One-shot alternative `gym eval run` starts the server stack, preprocesses, and -collects rollouts in a single command (don't run `gym env run` separately). +collects rollouts in a single command (don't run `gym env start` separately). Input path and agent ref are auto-derived from the `type: benchmark` dataset entry in the chained config: diff --git a/benchmarks/librispeech_pc/README.md b/benchmarks/librispeech_pc/README.md index b70bf6758c..676fe50743 100644 --- a/benchmarks/librispeech_pc/README.md +++ b/benchmarks/librispeech_pc/README.md @@ -43,7 +43,7 @@ and writes the JSONL into `benchmarks/librispeech_pc/data/`. ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark librispeech_pc ``` diff --git a/benchmarks/livecodebench-x/README.md b/benchmarks/livecodebench-x/README.md index 9bf1f83e34..9002d09f59 100644 --- a/benchmarks/livecodebench-x/README.md +++ b/benchmarks/livecodebench-x/README.md @@ -64,7 +64,7 @@ python benchmarks/livecodebench-x/prepare.py --prompt_language en ## Quickstart ```bash -gym env run \ +gym env start \ --benchmark livecodebench-x \ --model-type vllm_model ``` @@ -89,7 +89,7 @@ gym eval run --no-serve \ `--config` and `+prompt_config` are required: the prepared JSONL ships raw benchmark rows (no `responses_create_params.input` baked in), and the -agent's dataset-level `prompt_config` is metadata for `gym env run` only — the +agent's dataset-level `prompt_config` is metadata for `gym env start` only — the rollout CLI needs `+prompt_config=...` directly to apply the prompt template before merging `responses_create_params` overrides. `mkdir -p` is needed because `gym eval run --no-serve` does not create parent directories. diff --git a/benchmarks/longbench_v2/README.md b/benchmarks/longbench_v2/README.md index 75aae3081a..36b4233864 100644 --- a/benchmarks/longbench_v2/README.md +++ b/benchmarks/longbench_v2/README.md @@ -48,7 +48,7 @@ gym eval prepare --benchmark longbench_v2 gym eval prepare --benchmark longbench_v2/config_n3_1m # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark longbench_v2 diff --git a/benchmarks/longcodebench/README.md b/benchmarks/longcodebench/README.md index bf1e17d639..efb0e85ea2 100644 --- a/benchmarks/longcodebench/README.md +++ b/benchmarks/longcodebench/README.md @@ -42,7 +42,7 @@ gym eval prepare --benchmark longcodebench gym eval prepare --benchmark longcodebench/config_n3_1m # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark longcodebench diff --git a/benchmarks/m_arena_hard/README.md b/benchmarks/m_arena_hard/README.md index 54d1f1f784..89835f94b4 100644 --- a/benchmarks/m_arena_hard/README.md +++ b/benchmarks/m_arena_hard/README.md @@ -51,7 +51,7 @@ gym eval prepare --benchmark m_arena_hard python benchmarks/m_arena_hard/prepare.py --baseline-file path/to/baselines.jsonl # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark m_arena_hard diff --git a/benchmarks/m_arena_hard_v2/README.md b/benchmarks/m_arena_hard_v2/README.md index 1e8f2a5201..03e9805927 100644 --- a/benchmarks/m_arena_hard_v2/README.md +++ b/benchmarks/m_arena_hard_v2/README.md @@ -58,7 +58,7 @@ python benchmarks/m_arena_hard_v2/prepare.py \ gym eval prepare --benchmark m_arena_hard_v2 # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark m_arena_hard_v2 diff --git a/benchmarks/math-500/README.md b/benchmarks/math-500/README.md index 40df2cc50f..acac10600c 100644 --- a/benchmarks/math-500/README.md +++ b/benchmarks/math-500/README.md @@ -18,7 +18,7 @@ resource server. gym eval prepare --benchmark math-500 # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark math-500 diff --git a/benchmarks/mbpp/README.md b/benchmarks/mbpp/README.md index 5461104f2b..6b511968f1 100644 --- a/benchmarks/mbpp/README.md +++ b/benchmarks/mbpp/README.md @@ -25,14 +25,14 @@ Verification runs in the `evalplus` resource server (shared with gym eval prepare --benchmark mbpp # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark mbpp # Collecting rollouts. The +prompt_config= override is required because # the prepared JSONL stores raw `question` rows (no `responses_create_params.input`); # gym eval run --no-serve does not pick up the `prompt_config:` field on the dataset -# entry in config.yaml the way gym env run does. +# entry in config.yaml the way gym env start does. gym eval run --no-serve \ --agent mbpp_evalplus_simple_agent \ --input benchmarks/mbpp/data/mbpp_benchmark.jsonl \ diff --git a/benchmarks/minif2f/README.md b/benchmarks/minif2f/README.md index 9941d01a6e..f057727457 100644 --- a/benchmarks/minif2f/README.md +++ b/benchmarks/minif2f/README.md @@ -30,7 +30,7 @@ and set `NEMO_SKILLS_SANDBOX_HOST` / `NEMO_SKILLS_SANDBOX_PORT` before starting the server. ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark minif2f ``` diff --git a/benchmarks/mmlu-redux/README.md b/benchmarks/mmlu-redux/README.md index 8a63979880..422d0d3198 100644 --- a/benchmarks/mmlu-redux/README.md +++ b/benchmarks/mmlu-redux/README.md @@ -18,7 +18,7 @@ Migrates NeMo Skills' `mmlu-redux` benchmark to Gym on top of the shared gym eval prepare --benchmark mmlu-redux # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark mmlu-redux diff --git a/benchmarks/mmlu/README.md b/benchmarks/mmlu/README.md index 33fd5839a8..f4e0b88e8e 100644 --- a/benchmarks/mmlu/README.md +++ b/benchmarks/mmlu/README.md @@ -17,7 +17,7 @@ resource server. gym eval prepare --benchmark mmlu # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark mmlu diff --git a/benchmarks/mmlu_pro/README.md b/benchmarks/mmlu_pro/README.md index 2a0be5e720..901174b926 100644 --- a/benchmarks/mmlu_pro/README.md +++ b/benchmarks/mmlu_pro/README.md @@ -16,7 +16,7 @@ This benchmark uses the `mcqa` resource server with the `mcqa_simple_agent`. gym eval prepare --benchmark mmlu_pro # Start servers -gym env run \ +gym env start \ --benchmark mmlu_pro \ --model-type vllm_model diff --git a/benchmarks/mmlu_prox/README.md b/benchmarks/mmlu_prox/README.md index 72c5cab8f8..07ab5560f2 100644 --- a/benchmarks/mmlu_prox/README.md +++ b/benchmarks/mmlu_prox/README.md @@ -16,7 +16,7 @@ This benchmark uses the `mcqa` resource server with the `mcqa_simple_agent`. gym eval prepare --benchmark mmlu_prox # Start servers -gym env run \ +gym env start \ --benchmark mmlu_prox \ --model-type vllm_model diff --git a/benchmarks/mmmlu/README.md b/benchmarks/mmmlu/README.md index 93f80f5768..d62d933c0e 100644 --- a/benchmarks/mmmlu/README.md +++ b/benchmarks/mmmlu/README.md @@ -17,7 +17,7 @@ resource server. gym eval prepare --benchmark mmmlu # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark mmmlu diff --git a/benchmarks/mobench/README.md b/benchmarks/mobench/README.md index e2e9c91d19..3f43becd65 100644 --- a/benchmarks/mobench/README.md +++ b/benchmarks/mobench/README.md @@ -29,7 +29,7 @@ and set `NEMO_SKILLS_SANDBOX_HOST` / `NEMO_SKILLS_SANDBOX_PORT` before starting the server. ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark mobench ``` diff --git a/benchmarks/mrcr/README.md b/benchmarks/mrcr/README.md index 8e89886e2e..ebdfabeb38 100644 --- a/benchmarks/mrcr/README.md +++ b/benchmarks/mrcr/README.md @@ -45,7 +45,7 @@ gym eval prepare --benchmark mrcr/config_n3_1m ## Start environment ```bash -gym env run \ +gym env start \ --benchmark mrcr \ --model-type vllm_model ``` diff --git a/benchmarks/musan/README.md b/benchmarks/musan/README.md index 8a9f86f565..c317058685 100644 --- a/benchmarks/musan/README.md +++ b/benchmarks/musan/README.md @@ -53,7 +53,7 @@ points at `/data/musan//audio/musan__NNNNNN.wav` by default ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark musan ``` diff --git a/benchmarks/numb3rs/README.md b/benchmarks/numb3rs/README.md index 75e3b96fc3..239aa055a2 100644 --- a/benchmarks/numb3rs/README.md +++ b/benchmarks/numb3rs/README.md @@ -64,7 +64,7 @@ combined `benchmarks/numb3rs/data/numb3rs_benchmark.jsonl`. ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark numb3rs ``` diff --git a/benchmarks/physics/README.md b/benchmarks/physics/README.md index 413c09f7c0..6664b9f5cc 100644 --- a/benchmarks/physics/README.md +++ b/benchmarks/physics/README.md @@ -24,7 +24,7 @@ transformation Skills uses for the multi-part answers, and writes ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark physics ``` diff --git a/benchmarks/polymath/README.md b/benchmarks/polymath/README.md index fd711cb29a..e4620107b6 100644 --- a/benchmarks/polymath/README.md +++ b/benchmarks/polymath/README.md @@ -50,7 +50,7 @@ Start the servers (inherits the `polymath` resources server in symbolic-only mode plus a vLLM model server): ``` -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark polymath ``` diff --git a/benchmarks/proof-arena-judge/README.md b/benchmarks/proof-arena-judge/README.md index b87f4e7cea..09b3d132b0 100644 --- a/benchmarks/proof-arena-judge/README.md +++ b/benchmarks/proof-arena-judge/README.md @@ -20,7 +20,7 @@ Adds the `proof-arena-judge` benchmark to Gym. gym eval prepare --benchmark proof-arena-judge # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark proof-arena-judge diff --git a/benchmarks/proof_bench_judge/README.md b/benchmarks/proof_bench_judge/README.md index 38b4e6f8aa..3ff92536e5 100644 --- a/benchmarks/proof_bench_judge/README.md +++ b/benchmarks/proof_bench_judge/README.md @@ -37,7 +37,7 @@ details. gym eval prepare --benchmark proof_bench_judge # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark proof_bench_judge diff --git a/benchmarks/proofnet/README.md b/benchmarks/proofnet/README.md index 46ff2bd60a..9322ab3615 100644 --- a/benchmarks/proofnet/README.md +++ b/benchmarks/proofnet/README.md @@ -28,7 +28,7 @@ and set `NEMO_SKILLS_SANDBOX_HOST` / `NEMO_SKILLS_SANDBOX_PORT` before starting the server. ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark proofnet ``` diff --git a/benchmarks/putnam_bench/README.md b/benchmarks/putnam_bench/README.md index dc20c6c8ce..a286df8830 100644 --- a/benchmarks/putnam_bench/README.md +++ b/benchmarks/putnam_bench/README.md @@ -28,7 +28,7 @@ and set `NEMO_SKILLS_SANDBOX_HOST` / `NEMO_SKILLS_SANDBOX_PORT` before starting the server. ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark putnam_bench ``` diff --git a/benchmarks/simpleqa/README.md b/benchmarks/simpleqa/README.md index 7bb4512fdb..55230fc3ca 100644 --- a/benchmarks/simpleqa/README.md +++ b/benchmarks/simpleqa/README.md @@ -40,7 +40,7 @@ gym eval prepare --benchmark simpleqa ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark simpleqa ``` diff --git a/benchmarks/speed-bench/README.md b/benchmarks/speed-bench/README.md index e247232263..596b14eb95 100644 --- a/benchmarks/speed-bench/README.md +++ b/benchmarks/speed-bench/README.md @@ -57,7 +57,7 @@ gym eval prepare --benchmark speed-bench/config_qualitative # ngram speculative decoding into vllm_serve_kwargs.speculative_config. # To use a different target model, swap this for any local_vllm_model # config that includes a `speculative_config:` block. -gym env run \ +gym env start \ --model-type local_vllm_model/Qwen/Qwen3-30B-A3B-Instruct-2507-ngram-specdec \ --benchmark speed-bench/config_qualitative \ +policy_model=Qwen3-30B-A3B-Instruct-2507-ngram-specdec diff --git a/benchmarks/supergpqa/README.md b/benchmarks/supergpqa/README.md index 66c4462499..915cc4cc09 100644 --- a/benchmarks/supergpqa/README.md +++ b/benchmarks/supergpqa/README.md @@ -17,7 +17,7 @@ server. gym eval prepare --benchmark supergpqa # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark supergpqa diff --git a/benchmarks/ugphysics/README.md b/benchmarks/ugphysics/README.md index 1b1248307c..7c273675d0 100644 --- a/benchmarks/ugphysics/README.md +++ b/benchmarks/ugphysics/README.md @@ -64,7 +64,7 @@ message. gym eval prepare --benchmark ugphysics # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark ugphysics diff --git a/benchmarks/wmt24pp/README.md b/benchmarks/wmt24pp/README.md index 2b2839e11b..6d0f3e622f 100644 --- a/benchmarks/wmt24pp/README.md +++ b/benchmarks/wmt24pp/README.md @@ -32,7 +32,7 @@ runs disable COMET via Hydra override and rely on corpus-BLEU only; xCOMET scoring still works end-to-end on the cluster path: ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --benchmark wmt24pp \ "++wmt24pp_wmt_translation_resources_server.resources_servers.wmt_translation.compute_comet=false" diff --git a/environments/abstention/README.md b/environments/abstention/README.md index 7809edc87a..00418656b3 100644 --- a/environments/abstention/README.md +++ b/environments/abstention/README.md @@ -15,7 +15,7 @@ The dataset used is [HotPotQA](https://hotpotqa.github.io/) (fullwiki split). ## Running servers ```bash -gym env run \ +gym env start \ --environment abstention \ --model-type openai_model \ +abstention.resources_servers.abstention.judge_model_server.name=policy_model diff --git a/environments/arc_agi/README.md b/environments/arc_agi/README.md index 7fbe827092..84cb034fc1 100644 --- a/environments/arc_agi/README.md +++ b/environments/arc_agi/README.md @@ -51,7 +51,7 @@ uv sync ### Start ARC-AGI environment (we can reuse the same one for ARC-AGI-1 and 2): ```bash -gym env run --environment arc_agi --model-type vllm_model +gym env start --environment arc_agi --model-type vllm_model ``` diff --git a/environments/blackjack/README.md b/environments/blackjack/README.md index 2ef35c7628..bcbedeab64 100644 --- a/environments/blackjack/README.md +++ b/environments/blackjack/README.md @@ -9,7 +9,7 @@ Example data provided in `data/example.jsonl` (system prompt only, no verifier_m ## Run ```bash -gym env run --environment blackjack --model-type vllm_model +gym env start --environment blackjack --model-type vllm_model ``` ## Data diff --git a/environments/calendar/README.md b/environments/calendar/README.md index f16a5db21c..0431490fb1 100644 --- a/environments/calendar/README.md +++ b/environments/calendar/README.md @@ -16,7 +16,7 @@ Create an `env.yaml` file in the Gym root directory to specifying `policy_base_u The following is an example command for running this resources server along with an OpenAI model: ```bash -gym env run --environment calendar --model-type openai_model +gym env start --environment calendar --model-type openai_model ``` ## Collecting rollouts diff --git a/environments/calendar_v2/README.md b/environments/calendar_v2/README.md index 4e80000792..2f2586e06f 100644 --- a/environments/calendar_v2/README.md +++ b/environments/calendar_v2/README.md @@ -16,7 +16,7 @@ The conversations in the dataset are generated using personas from the [nvidia/N The following is an example command for running this resources server along with an OpenAI model: ```bash -gym env run --environment calendar_v2 --model-type openai_model +gym env start --environment calendar_v2 --model-type openai_model ``` ## Collecting rollouts diff --git a/environments/circle_click/README.md b/environments/circle_click/README.md index a748601093..eeb7a3c052 100644 --- a/environments/circle_click/README.md +++ b/environments/circle_click/README.md @@ -12,7 +12,7 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & -gym env run --environment circle_click --model-type vllm_model & +gym env start --environment circle_click --model-type vllm_model & gym eval run --no-serve --agent circle_click_simple_agent --input environments/circle_click/data/example.jsonl --output environments/circle_click/data/example_rollouts.jsonl --limit 1 ``` diff --git a/environments/circle_count/README.md b/environments/circle_count/README.md index 2af1986cd8..691e45fe54 100644 --- a/environments/circle_count/README.md +++ b/environments/circle_count/README.md @@ -19,7 +19,7 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & -gym env run --environment circle_count --model-type vllm_model & +gym env start --environment circle_count --model-type vllm_model & gym eval run --no-serve --agent circle_count_simple_agent --input environments/circle_count/data/example.jsonl --output environments/circle_count/data/example_rollouts.jsonl --limit 1 ``` diff --git a/environments/code_gen/README.md b/environments/code_gen/README.md index 3e5edf9b58..8a4b86fd4b 100644 --- a/environments/code_gen/README.md +++ b/environments/code_gen/README.md @@ -24,7 +24,7 @@ We use the LiveCodeBench execution code. Create an `env.yaml` file in the Gym root directory to specify the model endpoint and credentials. See [documentation](https://docs.nvidia.com/nemo/gym/reference/configuration#local-configuration-envyaml) for details. ```bash # Running the server -gym env run --environment code_gen --model-type openai_model +gym env start --environment code_gen --model-type openai_model # Collect rollouts from example problems gym eval run --no-serve --agent code_gen_simple_agent \ diff --git a/environments/ether0/README.md b/environments/ether0/README.md index d10191bf04..e7943d7b10 100644 --- a/environments/ether0/README.md +++ b/environments/ether0/README.md @@ -27,7 +27,7 @@ Start servers and collect rollouts ```bash # start vllm and nemo gym servers vllm serve futurehouse/ether0 & -gym env run --environment ether0 --model-type vllm_model & +gym env start --environment ether0 --model-type vllm_model & # wait for above to be ready gym eval run --no-serve \ diff --git a/environments/instruction_following/README.md b/environments/instruction_following/README.md index 97c20184b5..575073b910 100644 --- a/environments/instruction_following/README.md +++ b/environments/instruction_following/README.md @@ -47,7 +47,7 @@ The dataset can be found at https://huggingface.co/datasets/nvidia/Nemotron-RL-i ### Usage Create an `env.yaml` file in the Gym root directory to specifying `policy_base_url`, `policy_model_name`, and `policy_api_key`. See [documentation](https://docs.nvidia.com/nemo/gym/reference/configuration#local-configuration-envyaml) for details. ```bash -gym env run --environment instruction_following --model-type vllm_model +gym env start --environment instruction_following --model-type vllm_model python environments/instruction_following/prepare.py diff --git a/environments/reasoning_gym/README.md b/environments/reasoning_gym/README.md index f096069777..dac145fc39 100644 --- a/environments/reasoning_gym/README.md +++ b/environments/reasoning_gym/README.md @@ -87,7 +87,7 @@ policy_model_name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 ## Launch nemo gym servers ```bash -gym env run --environment reasoning_gym --model-type vllm_model +gym env start --environment reasoning_gym --model-type vllm_model ``` ## Collect rollouts diff --git a/environments/workplace_assistant/README.md b/environments/workplace_assistant/README.md index e21fc076a2..2be692c8bf 100644 --- a/environments/workplace_assistant/README.md +++ b/environments/workplace_assistant/README.md @@ -11,7 +11,7 @@ Commands - Spin up server: ``` -gym env run \ +gym env start \ --model-type openai_model \ --environment workplace_assistant ``` diff --git a/fern/versions/latest/pages/contribute/environments/new-environment.mdx b/fern/versions/latest/pages/contribute/environments/new-environment.mdx index 92470cdcf5..a57d1f71bc 100644 --- a/fern/versions/latest/pages/contribute/environments/new-environment.mdx +++ b/fern/versions/latest/pages/contribute/environments/new-environment.mdx @@ -80,7 +80,7 @@ Write and run tests for your resources server: Verify basic functionality and generate example rollouts: -- Document the command used to start your server, for example, `gym env run --resources-server my_server` +- Document the command used to start your server, for example, `gym env start --resources-server my_server` - Generate rollouts and save 5 example outputs to `data/example_rollouts.jsonl` to demonstrate correct reward signals ### 5. Reward Profiling diff --git a/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx b/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx index 4f123edf1c..e141c10cc9 100644 --- a/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx +++ b/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx @@ -413,12 +413,12 @@ pytest -v Start the servers: ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server my_weather_tool ``` -`gym env run` reads the config files and starts all three components from the architecture diagram: +`gym env start` reads the config files and starts all three components from the architecture diagram: 1. **Agent Server** (`my_weather_tool_simple_agent`) — the `simple_agent` that orchestrates the seed → model → tool → verify loop 2. **Model Server** (`openai_model`) — proxies LLM inference requests to the OpenAI API @@ -662,4 +662,4 @@ source .venv/bin/activate python app.py ``` -Server logs appear in the terminal where `gym env run` was executed. +Server logs appear in the terminal where `gym env start` was executed. diff --git a/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx b/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx index bec06225fb..bb787ed438 100644 --- a/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx +++ b/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx @@ -94,7 +94,7 @@ The resources server config points the judge at the policy model — `judge_mode The config file ships with a `judge_model` block that starts a dedicated judge server. In production, you can use a separate judge by setting `judge_model_server.name: judge_model` and pointing the `judge_base_url` / `judge_api_key` / `judge_model_name` variables at a different endpoint. This lets you use a different model, provider, or quota for the judge. -Since this walkthrough reuses `policy_model` as the judge, **comment out the `judge_model` block** as shown below — otherwise `gym env run` will start an unused server that still needs its variables to resolve. +Since this walkthrough reuses `policy_model` as the judge, **comment out the `judge_model` block** as shown below — otherwise `gym env start` will start an unused server that still needs its variables to resolve. Be sure to **set `judge_model_server.name` to `policy_model`** as well. @@ -200,7 +200,7 @@ If you are building your own LLM-judge server, you will write similar code — t Start the servers: ```bash -gym env run \ +gym env start \ --resources-server over_refusal_detection \ --model-type openai_model ``` @@ -350,7 +350,7 @@ Other servers apply the same pattern with domain-specific variations. For exampl 2. Add or reuse a **model server** for the judge; reference it from `judge_model_server`. 3. Design **prompts and parseable verdicts**; handle judge failures gracefully. 4. Set **temperature / max tokens** and **concurrency** for your SLA and budget. -5. Smoke-test with `gym env run` and your resources server's **`data/example.jsonl`**, then scale with `gym eval run --no-serve`. +5. Smoke-test with `gym env start` and your resources server's **`data/example.jsonl`**, then scale with `gym eval run --no-serve`. Done looks like: diff --git a/fern/versions/latest/pages/get-started/quickstart.mdx b/fern/versions/latest/pages/get-started/quickstart.mdx index bad7ef327a..819db0fdc9 100644 --- a/fern/versions/latest/pages/get-started/quickstart.mdx +++ b/fern/versions/latest/pages/get-started/quickstart.mdx @@ -31,7 +31,7 @@ Run your agent on a set of tasks and score the results. This example uses a simp NeMo Gym uses local servers to coordinate your model, agent, and task verification. Start them first: ```bash -gym env run \ +gym env start \ --resources-server mcqa \ --model-type openai_model ``` @@ -110,7 +110,7 @@ Every CLI command supports `-h` or `--help` for detailed usage information: ```bash gym --help -gym env run --help +gym env start --help ``` ## Next Steps diff --git a/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx b/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx index 55eacd2ac9..ca65afe6fa 100644 --- a/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx +++ b/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx @@ -6,7 +6,7 @@ This section describes how NeMo Gym components interact during startup and execu ## Control Plane: Server Startup -When you run `gym env run`, the system starts up in four phases: +When you run `gym env start`, the system starts up in four phases: ```mermaid %%{init: {'theme': 'default', 'themeVariables': { 'lineColor': '#5c6bc0', 'primaryTextColor': '#333', 'primaryColor': '#e3f2fd', 'secondaryColor': '#f5f5f5'}}}%% @@ -18,10 +18,10 @@ flowchart LR ### Phase 1: Parse CLI -The `gym env run` command uses Hydra to parse command-line arguments. Users specify configuration files via `--config`: +The `gym env start` command uses Hydra to parse command-line arguments. Users specify configuration files via `--config`: ```bash -gym env run \ +gym env start \ --resources-server math \ --model-type openai_model ``` @@ -47,7 +47,7 @@ Servers are started in two stages: ```mermaid %%{init: {'theme': 'default', 'themeVariables': { 'lineColor': '#5c6bc0', 'primaryTextColor': '#333'}}}%% flowchart TB - Main["Main Process
gym env run"] + Main["Main Process
gym env start"] Head["Head Server
Port 11000"] Model["Model Server
Own venv + port"] Agent["Agent Server
Own venv + port"] diff --git a/fern/versions/latest/pages/model-server/vllm.mdx b/fern/versions/latest/pages/model-server/vllm.mdx index 5b3cddd591..61f71f5993 100644 --- a/fern/versions/latest/pages/model-server/vllm.mdx +++ b/fern/versions/latest/pages/model-server/vllm.mdx @@ -11,7 +11,7 @@ VLLMModel provides a Responses API to Chat Completions mapping middleware layer **To use VLLMModel, just change the `responses_api_models/openai_model/configs/openai_model.yaml` in your config paths to `responses_api_models/vllm_model/configs/vllm_model.yaml`!** ```bash -gym env run \ +gym env start \ --resources-server example_multi_step \ --model-type vllm_model ``` @@ -116,7 +116,7 @@ vllm serve \ ### Configure NeMo Gym to use the local vLLM server In a second terminal on the same GPU node that was used to spin up the vLLM server, enter the NeMo Gym Python environment, and start the NeMo Gym servers. ```bash -gym env run \ +gym env start \ --resources-server example_multi_step \ --model-type vllm_model \ --model-url http://0.0.0.0:10240/v1 \ diff --git a/fern/versions/latest/pages/reference/cli-commands.mdx b/fern/versions/latest/pages/reference/cli-commands.mdx index 1c702245bd..76a44cf01c 100644 --- a/fern/versions/latest/pages/reference/cli-commands.mdx +++ b/fern/versions/latest/pages/reference/cli-commands.mdx @@ -36,7 +36,7 @@ gym env init # scaffold a new resources server gym env resolve # resolve and print the final merged config gym env packages # list packages in a server's virtual environment gym env test # test resources server(s); all of them if none is given -gym env run # start the servers +gym env start # start the servers gym env status # show running servers # Evaluation @@ -86,7 +86,7 @@ Unknown `--flags` (note the leading dashes) are treated as typos and rejected wi ### Selecting the model server -Commands that need a model (`gym env run`, `gym eval run`) configure it through four flags: +Commands that need a model (`gym env start`, `gym eval run`) configure it through four flags: | Flag | Description | | --- | --- | @@ -355,7 +355,7 @@ gym env test --resources-server example_single_tool_call gym env test ``` -### `gym env run` +### `gym env start` Start the NeMo Gym servers (agents, models, resources) defined by the provided configs. Reads configuration from YAML files and runs each configured server in its own environment. Replaces `ng_run`. @@ -371,12 +371,12 @@ Start the NeMo Gym servers (agents, models, resources) defined by the provided c | `--model-api-key` | Model server API key. | ```bash -gym env run \ +gym env start \ --resources-server example_single_tool_call \ --model-type openai_model # Start a benchmark's servers -gym env run \ +gym env start \ --benchmark gpqa \ --model-type vllm_model ``` @@ -431,7 +431,7 @@ gym eval prepare --benchmark aime24 ### `gym eval run` -Collate data, start the servers, and collect rollouts. This is the main evaluation command. By default it spins up all required servers (the former `ng_e2e_collect_rollouts`). Pass `--no-serve` to collect against servers you already started with `gym env run` (the former `ng_collect_rollouts`). Replaces `ng_e2e_collect_rollouts` and `ng_collect_rollouts`. +Collate data, start the servers, and collect rollouts. This is the main evaluation command. By default it spins up all required servers (the former `ng_e2e_collect_rollouts`). Pass `--no-serve` to collect against servers you already started with `gym env start` (the former `ng_collect_rollouts`). Replaces `ng_e2e_collect_rollouts` and `ng_collect_rollouts`. | Option | Description | | --- | --- | @@ -531,7 +531,7 @@ The legacy `ng_*` and `nemo_gym_*` console scripts are deprecated but still func | `ng_help` | `gym --help` | | `ng_version` | `gym --version` | | `ng_list_benchmarks` | `gym list benchmarks` | -| `ng_run` | `gym env run` | +| `ng_run` | `gym env start` | | `ng_status` | `gym env status` | | `ng_dump_config` | `gym env resolve` | | `ng_pip_list` | `gym env packages` | @@ -585,7 +585,7 @@ The most common Hydra overrides now have dedicated flags. Prefer flags; fall bac # Before: ng_run "+config_paths=[resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" # After: -gym env run \ +gym env start \ --resources-server example_single_tool_call \ --model-type openai_model diff --git a/fern/versions/latest/pages/reference/configuration.mdx b/fern/versions/latest/pages/reference/configuration.mdx index a005d9a23e..e07c4bf0c1 100644 --- a/fern/versions/latest/pages/reference/configuration.mdx +++ b/fern/versions/latest/pages/reference/configuration.mdx @@ -146,34 +146,34 @@ error_on_almost_servers: true # Exit on invalid configs (default: true) ## Command Line Usage -To run servers, use `gym env run`. NeMo Gym uses [Hydra](https://hydra.cc/) for configuration management. +To run servers, use `gym env start`. NeMo Gym uses [Hydra](https://hydra.cc/) for configuration management. ### Loading Configs ```bash # Load one or more config files -gym env run \ +gym env start \ --config config1.yaml \ --config config2.yaml # Use paths stored in env.yaml -gym env run "+config_paths=${my_config_paths}" +gym env start "+config_paths=${my_config_paths}" ``` ### Overriding Values ```bash # Override nested values (use dot notation after server ID) -gym env run \ +gym env start \ --config config.yaml \ +my_server.resources_servers.my_impl.domain=coding # Override policy model -gym env run --config config.yaml \ +gym env start --config config.yaml \ --model gpt-4o-mini # Disable strict validation -gym env run \ +gym env start \ --config config.yaml \ +error_on_almost_servers=false ``` diff --git a/fern/versions/latest/pages/reference/faq.mdx b/fern/versions/latest/pages/reference/faq.mdx index 1eac05b8b2..bf556fad9a 100644 --- a/fern/versions/latest/pages/reference/faq.mdx +++ b/fern/versions/latest/pages/reference/faq.mdx @@ -253,7 +253,7 @@ gym dataset collate \ Run NeMo Gym servers the exact same way with the same configs! ```bash -gym env run \ +gym env start \ --resources-server example_multi_step \ --model-type openai_model ``` @@ -291,7 +291,7 @@ In one terminal, start your agent, model, and resources servers, with profiling - `profiling_enabled` (bool): whether profiling is enabled or not. By default this is disabled since it incurs some slight overhead we don't want at runtime. - `profiling_results_dirpath` (str): The directory to save all server profiling results in. Previous logs for the same will be overwritten in the same directory. ```bash -gym env run \ +gym env start \ --model-type openai_model \ --config resources_servers/math_with_judge/configs/bytedtsinghua_dapo17k.yaml \ +profiling_enabled=true \ @@ -341,7 +341,7 @@ NeMo Gym automatically sets up Ray for distributed computing for CPU-intensive t Ray is initialized when you start NeMo Gym servers: ```bash -gym env run --config $config_paths +gym env start --config $config_paths ``` The initialization happens in two places: @@ -400,7 +400,7 @@ Tying back to NeMo Gym, NeMo Gym can be used to create synthetic data for SFT tr # FAQ: PermissionError when starting NeMo Gym in sandboxed environments -If you see an error like the following when running `gym env run`: +If you see an error like the following when running `gym env start`: ```python PermissionError: [Errno 1] Operation not permitted (originated from sysctl() malloc 1/3) @@ -431,7 +431,7 @@ If you're running NeMo Gym in a sandboxed environment and hit this error, you'll 2. **Grant additional permissions** to allow Ray to access process information 3. **Run outside the sandbox** in a normal terminal environment -For most development and production use cases, simply running `gym env run` in your regular terminal will work without any issues. +For most development and production use cases, simply running `gym env start` in your regular terminal will work without any issues. **Specific workaround for Cursor AI:** diff --git a/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx b/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx index 5571bb6c68..26a170e515 100644 --- a/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx +++ b/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx @@ -13,21 +13,21 @@ Suppose you want to use both the example_single_tool_call and example_multi_step For example_single_tool_call: ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server example_single_tool_call ``` For example_multi_step: ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server example_multi_step ``` To use both environments, add the YAML configs together as follows: ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server example_single_tool_call \ --resources-server example_multi_step diff --git a/fern/versions/latest/pages/troubleshooting/configuration.mdx b/fern/versions/latest/pages/troubleshooting/configuration.mdx index d52a530db7..e6f6fe8df4 100644 --- a/fern/versions/latest/pages/troubleshooting/configuration.mdx +++ b/fern/versions/latest/pages/troubleshooting/configuration.mdx @@ -3,7 +3,7 @@ title: "Configuration Errors" description: "" position: 1 --- -These errors appear when running `gym env run` or `gym eval run --no-serve`. NeMo Gym validates configuration at startup before launching servers. +These errors appear when running `gym env start` or `gym eval run --no-serve`. NeMo Gym validates configuration at startup before launching servers. See the [Configuration reference](/reference/configuration) for complete configuration syntax and options. @@ -33,7 +33,7 @@ policy_api_key: sk-your-api-key Or pass via command line: ```bash -gym env run \ +gym env start \ --config config.yaml \ --model-api-key sk-your-api-key ``` @@ -57,7 +57,7 @@ in the list of available servers: [ResourcesServerRef(...), ...] 2. Ensure all required config files are passed with `--config`: ```bash -gym env run \ +gym env start \ --config model.yaml \ --config resource.yaml \ --config agent.yaml @@ -102,7 +102,7 @@ error_on_almost_servers: false ```bash # Or via command line -gym env run \ +gym env start \ --config config.yaml \ +error_on_almost_servers=false ``` diff --git a/nemo_gym/cli/legacy.py b/nemo_gym/cli/legacy.py index ba11c8eed9..59c5dab631 100644 --- a/nemo_gym/cli/legacy.py +++ b/nemo_gym/cli/legacy.py @@ -28,7 +28,7 @@ # Legacy command (ng_/nemo_gym_ prefix stripped) -> equivalent `gym` subcommand tokens. LEGACY = { - "run": ["env", "run"], + "run": ["env", "start"], "test": ["env", "test"], "test_all": ["env", "test"], "dev_test": ["dev", "test"], diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index a2d425f113..d3917f5a52 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -115,7 +115,7 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: ) ) -# Shared model-server flags. Reused by commands that spin up / target a model server (`eval run`, `env run`). +# Shared model-server flags. Reused by commands that spin up / target a model server (`eval run`, `env start`). # --model is the served model identifier across all backends: an API model name, an HF id, or a local checkpoint # path, interpreted per --model-type (e.g. a path/HF id to serve with local_vllm_model). MODEL = _value_flag( @@ -407,7 +407,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: summary="Test the resources server(s); runs all if no resources server is given.", flags=(RESOURCES_SERVER,), ), - "env run": Command( + "env start": Command( target="nemo_gym.cli.env:run", summary="Start the servers.", flags=( diff --git a/nemo_gym/server_status.py b/nemo_gym/server_status.py index b4166cb94d..2ab6b05a85 100644 --- a/nemo_gym/server_status.py +++ b/nemo_gym/server_status.py @@ -78,7 +78,7 @@ def discover_servers(self) -> List[ServerInstanceDisplayConfig]: except (requests.RequestException, ConnectionError) as e: logger.warning( - "Could not connect to head server: %s. Is the head server running? Start it with: `gym env run`", + "Could not connect to head server: %s. Is the head server running? Start it with: `gym env start`", e, ) return [] diff --git a/resources_servers/abstention/README.md b/resources_servers/abstention/README.md index eb52885379..a7e11c9851 100644 --- a/resources_servers/abstention/README.md +++ b/resources_servers/abstention/README.md @@ -15,7 +15,7 @@ The dataset used is [HotPotQA](https://hotpotqa.github.io/) (fullwiki split). ## Running servers ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server abstention \ +abstention.resources_servers.abstention.judge_model_server.name=policy_model diff --git a/resources_servers/arc_agi/README.md b/resources_servers/arc_agi/README.md index 0347501602..463f058517 100644 --- a/resources_servers/arc_agi/README.md +++ b/resources_servers/arc_agi/README.md @@ -51,7 +51,7 @@ uv sync ### Start ARC-AGI environment (we can reuse the same one for ARC-AGI-1 and 2): ```bash -gym env run \ +gym env start \ --resources-server arc_agi \ --model-type vllm_model ``` diff --git a/resources_servers/arena_judge/README.md b/resources_servers/arena_judge/README.md index 406c5e909b..65fcb175b7 100644 --- a/resources_servers/arena_judge/README.md +++ b/resources_servers/arena_judge/README.md @@ -47,7 +47,7 @@ Each JSONL row must carry the following top-level fields (pydantic ```bash # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server arena_judge diff --git a/resources_servers/asr_with_pc/README.md b/resources_servers/asr_with_pc/README.md index 58c2019724..cb9d0e56b0 100644 --- a/resources_servers/asr_with_pc/README.md +++ b/resources_servers/asr_with_pc/README.md @@ -50,7 +50,7 @@ workaround until the schema is extended. ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server asr_with_pc ``` diff --git a/resources_servers/aviary/README.md b/resources_servers/aviary/README.md index b08a9e10fc..efe706ff5b 100644 --- a/resources_servers/aviary/README.md +++ b/resources_servers/aviary/README.md @@ -22,7 +22,7 @@ This resources server adapts [Aviary environments](https://github.com/Future-Hou Run the GSM8K Aviary resources server together with a model config: ```bash -gym env run \ +gym env start \ --resources-server aviary/gsm8k_aviary \ --model-type vllm_model ``` @@ -49,7 +49,7 @@ Then, prepare your Gym data with the task_idx values of the problems you would l Once the dataset server is running and is accessible at a specific URL, update your config based on [configs/bbh_remote.yaml](configs/bbh_remote.yaml) with the server URL and api key, and launch NeMo-Gym as follows: ```bash -gym env run \ +gym env start \ --resources-server aviary/bbh_remote \ --model-type vllm_model ``` @@ -100,7 +100,7 @@ cd /path/to/gym/directory ``` And then bring up NeMo-Gym: ```bash -gym env run \ +gym env start \ --resources-server aviary/bbh_bundled \ --model-type vllm_model ``` diff --git a/resources_servers/bigcodebench/README.md b/resources_servers/bigcodebench/README.md index d35f44967a..5948f09433 100644 --- a/resources_servers/bigcodebench/README.md +++ b/resources_servers/bigcodebench/README.md @@ -23,7 +23,7 @@ starts are instant. ```bash # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server bigcodebench diff --git a/resources_servers/bird_sql/README.md b/resources_servers/bird_sql/README.md index d8038466e0..78b7b10a6d 100644 --- a/resources_servers/bird_sql/README.md +++ b/resources_servers/bird_sql/README.md @@ -34,7 +34,7 @@ ensure_bird_sql() ### Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server bird_sql ``` diff --git a/resources_servers/blackjack/README.md b/resources_servers/blackjack/README.md index caf8af8f2b..aa724dcd21 100644 --- a/resources_servers/blackjack/README.md +++ b/resources_servers/blackjack/README.md @@ -9,7 +9,7 @@ Example data provided in `data/example.jsonl` (system prompt only, no verifier_m ## Run ```bash -gym env run \ +gym env start \ --resources-server blackjack \ --model-type vllm_model ``` diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index 407c81ffe9..2e718b0a56 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -29,7 +29,7 @@ judge_model_name: Qwen/Qwen3-235B-A22B-Instruct-2507 ```bash # If you want to run with browsecomp benchmark instead of the example samples, need to change the datasets part to the one like `benchmarks/browsecomp/config.yaml` in `resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml` -gym env run \ +gym env start \ --resources-server browsecomp_advanced_harness \ --model-type vllm_model ``` diff --git a/resources_servers/calendar/README.md b/resources_servers/calendar/README.md index 34279ea4e5..07ef641f5d 100644 --- a/resources_servers/calendar/README.md +++ b/resources_servers/calendar/README.md @@ -16,7 +16,7 @@ The conversations in the dataset are generated using personas from the [nvidia/N The following is an example command for running this resources server along with an OpenAI model: ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server calendar ``` diff --git a/resources_servers/circle_click/README.md b/resources_servers/circle_click/README.md index 53815c8a76..610dc63eb8 100644 --- a/resources_servers/circle_click/README.md +++ b/resources_servers/circle_click/README.md @@ -12,7 +12,7 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & -gym env run \ +gym env start \ --resources-server circle_click \ --model-type vllm_model & gym eval run --no-serve \ diff --git a/resources_servers/circle_count/README.md b/resources_servers/circle_count/README.md index 72f6c87352..e30908513e 100644 --- a/resources_servers/circle_count/README.md +++ b/resources_servers/circle_count/README.md @@ -19,7 +19,7 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & -gym env run \ +gym env start \ --resources-server circle_count \ --model-type vllm_model & gym eval run --no-serve \ diff --git a/resources_servers/code_fim/README.md b/resources_servers/code_fim/README.md index e55c4595dd..5e89306bc7 100644 --- a/resources_servers/code_fim/README.md +++ b/resources_servers/code_fim/README.md @@ -58,7 +58,7 @@ the reward; pass@k / majority@k are computed by the metrics layer. ```bash # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server code_fim diff --git a/resources_servers/code_gen/README.md b/resources_servers/code_gen/README.md index 7af636f064..fc0f0dff43 100644 --- a/resources_servers/code_gen/README.md +++ b/resources_servers/code_gen/README.md @@ -24,7 +24,7 @@ We use the LiveCodeBench execution code. ```bash # Running the server -gym env run \ +gym env start \ --model-type openai_model \ --resources-server code_gen diff --git a/resources_servers/competitive_coding_challenges/README.md b/resources_servers/competitive_coding_challenges/README.md index 7dfce341d1..962d8e125b 100644 --- a/resources_servers/competitive_coding_challenges/README.md +++ b/resources_servers/competitive_coding_challenges/README.md @@ -61,7 +61,7 @@ elsewhere. Cluster/SLURM users can co-launch the sandbox via Skills' The following allows users to launch the environment followed by the request for generating and collecting rollouts. ```bash -gym env run \ +gym env start \ --config "$config_paths" \ +simple_agent.responses_api_agents.simple_agent.resources_server.name=competitive_coding_challenges_resources_server diff --git a/resources_servers/cvdp/README.md b/resources_servers/cvdp/README.md index e6c3320b58..8df29b144d 100644 --- a/resources_servers/cvdp/README.md +++ b/resources_servers/cvdp/README.md @@ -123,7 +123,7 @@ apt install -y ./apptainer_1.3.1_amd64.deb ### Step 1 — Start servers ```bash -gym env run \ +gym env start \ --resources-server cvdp \ --model-type vllm_model ``` diff --git a/resources_servers/equivalence_llm_judge/README.md b/resources_servers/equivalence_llm_judge/README.md index e4c0974b69..a7881a27a7 100644 --- a/resources_servers/equivalence_llm_judge/README.md +++ b/resources_servers/equivalence_llm_judge/README.md @@ -51,7 +51,7 @@ equivalence_llm_judge_simple_agent: ### Usage Spin up with a judge model and prompt: ```bash -gym env run \ +gym env start \ --resources-server equivalence_llm_judge \ --model-type openai_model \ +equivalence_llm_judge.resources_servers.equivalence_llm_judge.judge_responses_create_params.max_output_tokens=256 \ diff --git a/resources_servers/ether0/README.md b/resources_servers/ether0/README.md index 8ab3d56419..38a43eff8b 100644 --- a/resources_servers/ether0/README.md +++ b/resources_servers/ether0/README.md @@ -27,7 +27,7 @@ Start servers and collect rollouts ```bash # start vllm and nemo gym servers vllm serve futurehouse/ether0 & -gym env run \ +gym env start \ --resources-server ether0 \ --model-type vllm_model & diff --git a/resources_servers/evalplus/README.md b/resources_servers/evalplus/README.md index 64421b3b5a..f07b850e58 100644 --- a/resources_servers/evalplus/README.md +++ b/resources_servers/evalplus/README.md @@ -46,7 +46,7 @@ compute pass@k for each separately. ```bash # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server evalplus diff --git a/resources_servers/example_multi_turn_gymnasium/README.md b/resources_servers/example_multi_turn_gymnasium/README.md index 2b7bc95d29..6d71a88f1a 100644 --- a/resources_servers/example_multi_turn_gymnasium/README.md +++ b/resources_servers/example_multi_turn_gymnasium/README.md @@ -15,7 +15,7 @@ Example data provided in `data/example.jsonl`. ## Run ```bash -gym env run \ +gym env start \ --resources-server example_multi_turn_gymnasium \ --model-type vllm_model ``` diff --git a/resources_servers/finance_sec_search/README.md b/resources_servers/finance_sec_search/README.md index 9ecbfd5ecb..46958307b0 100644 --- a/resources_servers/finance_sec_search/README.md +++ b/resources_servers/finance_sec_search/README.md @@ -152,7 +152,7 @@ public benchmark lives in `benchmarks/finance_sec_search/`. It downloads the `public.csv` dataset from GitHub and converts it to Gym format: ```bash -# Prepare via Gym CLI (recommended — used by gym env run with benchmark configs): +# Prepare via Gym CLI (recommended — used by gym env start with benchmark configs): gym eval prepare --benchmark finance_sec_search/config_no_web_search # Or run the script directly: @@ -194,7 +194,7 @@ Launch a vLLM-compatible model server (e.g. Qwen3-30B-A3B) so the policy and jud With a local vLLM model server: ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server finance_sec_search ``` @@ -202,7 +202,7 @@ gym env run \ Or with an OpenAI-compatible API (e.g. OpenAI, Azure, NIM): ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server finance_sec_search ``` diff --git a/resources_servers/format_verification/README.md b/resources_servers/format_verification/README.md index 78c9d48921..ade87834f6 100644 --- a/resources_servers/format_verification/README.md +++ b/resources_servers/format_verification/README.md @@ -26,7 +26,7 @@ See [ARCHITECTURE.md](ARCHITECTURE.md) for system diagrams and detailed field ma ### Freeform Formatting ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server format_verification/freeform_formatting ``` @@ -43,7 +43,7 @@ gym eval run --no-serve \ ### Citation Format ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server format_verification/citation_format ``` diff --git a/resources_servers/frontierscience_judge/README.md b/resources_servers/frontierscience_judge/README.md index 00823b48e3..890c19b1c3 100644 --- a/resources_servers/frontierscience_judge/README.md +++ b/resources_servers/frontierscience_judge/README.md @@ -35,7 +35,7 @@ judge's full text), and in research mode `rubric_score` plus ```bash # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server frontierscience_judge diff --git a/resources_servers/google_search/README.md b/resources_servers/google_search/README.md index fb8202c1ec..b435edf227 100644 --- a/resources_servers/google_search/README.md +++ b/resources_servers/google_search/README.md @@ -98,7 +98,7 @@ tools=[ ### Running the Server ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server google_search diff --git a/resources_servers/gpqa_diamond/README.md b/resources_servers/gpqa_diamond/README.md index 7530ed2d54..55dda32ce8 100644 --- a/resources_servers/gpqa_diamond/README.md +++ b/resources_servers/gpqa_diamond/README.md @@ -77,7 +77,7 @@ server. Using a local Nemotron 3 model with `local_vllm_model`: ```bash -gym env run \ +gym env start \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type local_vllm_model/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 \ --resources-server gpqa_diamond \ @@ -90,7 +90,7 @@ gym env run \ Generic example with `openai_model`: ```bash -gym env run \ +gym env start \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type openai_model \ --resources-server gpqa_diamond \ diff --git a/resources_servers/graphwalks/README.md b/resources_servers/graphwalks/README.md index 579e39efe0..cce3740a62 100644 --- a/resources_servers/graphwalks/README.md +++ b/resources_servers/graphwalks/README.md @@ -25,7 +25,7 @@ https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/evaluation/evaluator ## Start environment ```bash -gym env run \ +gym env start \ --resources-server graphwalks \ --model-type vllm_model ``` diff --git a/resources_servers/grl_sokoban/README.md b/resources_servers/grl_sokoban/README.md index f562787ce8..efd1bd6036 100644 --- a/resources_servers/grl_sokoban/README.md +++ b/resources_servers/grl_sokoban/README.md @@ -11,7 +11,7 @@ Single-box Sokoban puzzle environment. The environment is implemented under `res Spin up the server alongside a compatible agent: ```bash -gym env run \ +gym env start \ --model-type openai_model \ --config responses_api_agents/gymnasium_agent/configs/gymnasium_agent.yaml \ --resources-server grl_sokoban diff --git a/resources_servers/grl_tetris/README.md b/resources_servers/grl_tetris/README.md index a2b469814b..50792bb867 100644 --- a/resources_servers/grl_tetris/README.md +++ b/resources_servers/grl_tetris/README.md @@ -12,7 +12,7 @@ GRL Tetris environment in Gymnasium style. The model emits one or more ` Start NeMo Gym servers ```bash -gym env run \ +gym env start \ --model-type openai_model \ --config responses_api_agents/gymnasium_agent/configs/gymnasium_agent.yaml \ --resources-server grl_tetris diff --git a/resources_servers/hotpotqa_qa/README.md b/resources_servers/hotpotqa_qa/README.md index 2161aa182a..2f3224e13c 100644 --- a/resources_servers/hotpotqa_qa/README.md +++ b/resources_servers/hotpotqa_qa/README.md @@ -30,7 +30,7 @@ pass@k for each channel. ## Running servers ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server hotpotqa_qa ``` diff --git a/resources_servers/ifbench/README.md b/resources_servers/ifbench/README.md index 61f3800bcf..f2e4a48b0e 100644 --- a/resources_servers/ifbench/README.md +++ b/resources_servers/ifbench/README.md @@ -45,7 +45,7 @@ This clones the repo, applies patches, and writes the `.installed` marker. On su ## Example Usage ```bash -gym env run \ +gym env start \ --benchmark ifbench \ --model-type openai_model diff --git a/resources_servers/imo_gradingbench/README.md b/resources_servers/imo_gradingbench/README.md index 6e357071b3..93013666a8 100644 --- a/resources_servers/imo_gradingbench/README.md +++ b/resources_servers/imo_gradingbench/README.md @@ -68,7 +68,7 @@ grade. ```bash # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server imo_gradingbench diff --git a/resources_servers/imo_proofbench_judge/README.md b/resources_servers/imo_proofbench_judge/README.md index 9bc96b93d4..3d2db3b890 100644 --- a/resources_servers/imo_proofbench_judge/README.md +++ b/resources_servers/imo_proofbench_judge/README.md @@ -22,7 +22,7 @@ Use ``benchmarks/imo_proofbench/`` for the IMO-ProofBench dataset. ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server imo_proofbench_judge \ +judge_base_url=https://generativelanguage.googleapis.com/v1beta/openai \ diff --git a/resources_servers/instruction_following/README.md b/resources_servers/instruction_following/README.md index 9b99029921..7e184b5773 100644 --- a/resources_servers/instruction_following/README.md +++ b/resources_servers/instruction_following/README.md @@ -47,7 +47,7 @@ The dataset can be found at https://huggingface.co/datasets/nvidia/Nemotron-RL-i ### Usage ```bash -gym env run \ +gym env start \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type openai_model \ --resources-server instruction_following \ diff --git a/resources_servers/inverse_if/README.md b/resources_servers/inverse_if/README.md index 88bd7f8f12..64cf4a1f6e 100644 --- a/resources_servers/inverse_if/README.md +++ b/resources_servers/inverse_if/README.md @@ -9,7 +9,7 @@ Evaluates model responses on the **Inverse IF** (Instruction Following) benchmar gym env test --resources-server inverse_if # 2. Start servers (in terminal 1) -gym env run \ +gym env start \ --resources-server inverse_if \ --model-type vllm_model @@ -118,7 +118,7 @@ Tests cover: 1. **Start servers**: ```bash - gym env run \ + gym env start \ --resources-server inverse_if \ --model-type vllm_model ``` diff --git a/resources_servers/jailbreak_detection/README.md b/resources_servers/jailbreak_detection/README.md index 86f827d005..122dee20bd 100644 --- a/resources_servers/jailbreak_detection/README.md +++ b/resources_servers/jailbreak_detection/README.md @@ -27,7 +27,7 @@ The example dataset includes various jailbreak attack patterns: 1. Start the servers: ```bash -gym env run \ +gym env start \ --resources-server jailbreak_detection/jailbreak_detection_nemotron_combined_reward_tp8 \ --model-type openai_model \ --resources-server jailbreak_detection/safety_judge_model diff --git a/resources_servers/labbench2_vlm/README.md b/resources_servers/labbench2_vlm/README.md index afe5c34bb7..2eddbab829 100644 --- a/resources_servers/labbench2_vlm/README.md +++ b/resources_servers/labbench2_vlm/README.md @@ -178,7 +178,7 @@ putting them in `env.yaml`. Start the servers: ```bash -gym env run \ +gym env start \ --resources-server labbench2_vlm \ --resources-server labbench2_vlm/judge_model_openai \ --model-type openai_model @@ -224,7 +224,7 @@ rollout output). Regenerate it locally with the command above. ### One-shot alternative `gym eval run` starts the server stack, preprocesses, and collects -rollouts in a single command (don't run `gym env run` separately). Input path and +rollouts in a single command (don't run `gym env start` separately). Input path and agent ref are auto-derived from the dataset entry in the chained config (`--split` picks which one): diff --git a/resources_servers/longmt_eval/README.md b/resources_servers/longmt_eval/README.md index f35cfe6b2b..69f4f3c7e7 100644 --- a/resources_servers/longmt_eval/README.md +++ b/resources_servers/longmt_eval/README.md @@ -158,7 +158,7 @@ rollouts file. ```bash # Start servers (smoke-test mode — no GPU needed for the verifier) -gym env run \ +gym env start \ --resources-server longmt_eval \ --model-type vllm_model & \ ++longmt_eval.resources_servers.longmt_eval.compute_segale=false diff --git a/resources_servers/math_advanced_calculations/README.md b/resources_servers/math_advanced_calculations/README.md index fa4593d88d..3a5c7c90a4 100644 --- a/resources_servers/math_advanced_calculations/README.md +++ b/resources_servers/math_advanced_calculations/README.md @@ -8,7 +8,7 @@ Commands - Spin up server: ``` -gym env run \ +gym env start \ --model-type openai_model \ --resources-server math_advanced_calculations ``` diff --git a/resources_servers/math_proof_judgement/README.md b/resources_servers/math_proof_judgement/README.md index 2c9aca9040..3a395eb777 100644 --- a/resources_servers/math_proof_judgement/README.md +++ b/resources_servers/math_proof_judgement/README.md @@ -66,7 +66,7 @@ parser name that matches your model's reasoning tokens (see `vllm serve ```bash # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server math_proof_judgement diff --git a/resources_servers/math_with_autograder/README.md b/resources_servers/math_with_autograder/README.md index aa310e9cd9..75ad3030a3 100644 --- a/resources_servers/math_with_autograder/README.md +++ b/resources_servers/math_with_autograder/README.md @@ -35,7 +35,7 @@ existing math_with_judge consumers. ## Run servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server math_with_autograder \ --resources-server math_with_autograder/judge_gptoss20b diff --git a/resources_servers/math_with_code/README.md b/resources_servers/math_with_code/README.md index aab5e33323..85bf7a9472 100644 --- a/resources_servers/math_with_code/README.md +++ b/resources_servers/math_with_code/README.md @@ -105,7 +105,7 @@ gym dataset download --storage gitlab \ Start server ``` -gym env run \ +gym env start \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type openai_model \ --resources-server math_with_code \ diff --git a/resources_servers/math_with_judge/README.md b/resources_servers/math_with_judge/README.md index 50bb775348..97d1cd66ca 100644 --- a/resources_servers/math_with_judge/README.md +++ b/resources_servers/math_with_judge/README.md @@ -11,7 +11,7 @@ on Hugging Face. ## Running servers The following are example commands for running this resources server, along with the simple agent and an OpenAI model: ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server math_with_judge \ +math_with_judge.resources_servers.math_with_judge.judge_model_server.name=policy_model diff --git a/resources_servers/mcqa/README.md b/resources_servers/mcqa/README.md index 8b278f7ffb..909bdfaf48 100644 --- a/resources_servers/mcqa/README.md +++ b/resources_servers/mcqa/README.md @@ -111,7 +111,7 @@ For datasets with custom prompt formats, you can optionally use `template_metada **Standard format (with `grading_mode`):** ```bash -gym env run \ +gym env start \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type openai_model \ --resources-server mcqa \ diff --git a/resources_servers/mrcr/README.md b/resources_servers/mrcr/README.md index 18c138e6a7..38926e346d 100644 --- a/resources_servers/mrcr/README.md +++ b/resources_servers/mrcr/README.md @@ -27,7 +27,7 @@ always fails. ## Start environment ```bash -gym env run \ +gym env start \ --resources-server mrcr \ --model-type vllm_model ``` diff --git a/resources_servers/multichallenge/README.md b/resources_servers/multichallenge/README.md index b475e6ff4b..8a5c93fc8f 100644 --- a/resources_servers/multichallenge/README.md +++ b/resources_servers/multichallenge/README.md @@ -9,7 +9,7 @@ Evaluates model responses on the **MultiChallenge** benchmark using an LLM judge gym env test --resources-server multichallenge # 2. Start servers (in terminal 1) -gym env run \ +gym env start \ --resources-server multichallenge \ --model-type vllm_model @@ -106,7 +106,7 @@ Tests cover: 1. **Start servers**: ```bash - gym env run \ + gym env start \ --resources-server multichallenge \ --model-type vllm_model ``` diff --git a/resources_servers/newton_bench/README.md b/resources_servers/newton_bench/README.md index 4e4228d5df..efcd34edd7 100644 --- a/resources_servers/newton_bench/README.md +++ b/resources_servers/newton_bench/README.md @@ -91,7 +91,7 @@ vllm serve \ ### Launch servers ```bash -gym env run \ +gym env start \ --resources-server newton_bench \ --model-type vllm_model ``` diff --git a/resources_servers/ns_tools/README.md b/resources_servers/ns_tools/README.md index 47d2999738..eac6eff1f0 100644 --- a/resources_servers/ns_tools/README.md +++ b/resources_servers/ns_tools/README.md @@ -8,7 +8,7 @@ It integrates with the NeMo Skills ToolManager to dynamically load and execute t ## Running servers The following are example commands for running this resources server with the simple agent and a vLLM model: ```bash -gym env run \ +gym env start \ --resources-server ns_tools \ --resources-server math_with_judge \ --model-type vllm_model \ diff --git a/resources_servers/openenv/README.md b/resources_servers/openenv/README.md index 158dc62a5b..8253a2e89f 100644 --- a/resources_servers/openenv/README.md +++ b/resources_servers/openenv/README.md @@ -36,7 +36,7 @@ policy_model_name: Running an environment is a **two-step process**: -1. **Start the servers** with `gym env run` — this launches the resources server, agent server, and model server. Wait until you see `All 3 / 3 servers ready!`. +1. **Start the servers** with `gym env start` — this launches the resources server, agent server, and model server. Wait until you see `All 3 / 3 servers ready!`. 2. **Collect rollouts** with `gym eval run --no-serve` in a **separate terminal** — this sends the JSONL prompts through the agent, which calls the LLM and environment in a loop, and writes trajectories with rewards to an output file. #### Echo @@ -44,7 +44,7 @@ Running an environment is a **two-step process**: ```bash # Terminal 1: Start servers source .venv/bin/activate -gym env run \ +gym env start \ --resources-server openenv/openenv_echo \ --model-type openai_model @@ -62,7 +62,7 @@ gym eval run --no-serve \ ```bash # Terminal 1: Start servers source .venv/bin/activate -gym env run \ +gym env start \ --resources-server openenv/openenv_coding \ --model-type openai_model @@ -80,7 +80,7 @@ gym eval run --no-serve \ ```bash # Terminal 1: Start servers source .venv/bin/activate -gym env run \ +gym env start \ --resources-server openenv/openenv_maze \ --model-type openai_model @@ -184,7 +184,7 @@ Then start servers and collect rollouts: ```bash # Terminal 1: Start servers source .venv/bin/activate -gym env run \ +gym env start \ --config resources_servers/openenv/configs/openenv_.yaml \ --model-type openai_model diff --git a/resources_servers/physics_judge/README.md b/resources_servers/physics_judge/README.md index 04e268ae2b..05e0b9cf40 100644 --- a/resources_servers/physics_judge/README.md +++ b/resources_servers/physics_judge/README.md @@ -32,7 +32,7 @@ existing math_with_judge consumers (`aime24`, `aime25`, `gsm8k`, ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server physics_judge \ --resources-server physics_judge/judge_openai diff --git a/resources_servers/polymath/README.md b/resources_servers/polymath/README.md index cb2813946e..b25c6d6633 100644 --- a/resources_servers/polymath/README.md +++ b/resources_servers/polymath/README.md @@ -30,7 +30,7 @@ PolyMath additions are at the metric-aggregation layer: ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server polymath ``` diff --git a/resources_servers/rdkit_chemistry/README.md b/resources_servers/rdkit_chemistry/README.md index 241d63e9dc..cb10fe97d8 100644 --- a/resources_servers/rdkit_chemistry/README.md +++ b/resources_servers/rdkit_chemistry/README.md @@ -65,7 +65,7 @@ takes precedence over `use_box_format`. ## Example Usage ```bash -gym env run \ +gym env start \ --resources-server rdkit_chemistry \ --model-type openai_model diff --git a/resources_servers/reasoning_gym/README.md b/resources_servers/reasoning_gym/README.md index 2dbf312c26..f206b49947 100644 --- a/resources_servers/reasoning_gym/README.md +++ b/resources_servers/reasoning_gym/README.md @@ -82,7 +82,7 @@ policy_model_name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 ## Launch nemo gym servers ```bash -gym env run \ +gym env start \ --resources-server reasoning_gym \ --model-type vllm_model ``` diff --git a/resources_servers/simpleqa/README.md b/resources_servers/simpleqa/README.md index c600aa0581..71a8492b31 100644 --- a/resources_servers/simpleqa/README.md +++ b/resources_servers/simpleqa/README.md @@ -39,7 +39,7 @@ the reasoning trace is split off before the response reaches this server. ## Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server simpleqa \ +simpleqa.resources_servers.simpleqa.judge_model_server.name=policy_model diff --git a/resources_servers/single_step_tool_use_with_argument_comparison/README.md b/resources_servers/single_step_tool_use_with_argument_comparison/README.md index c225512251..ae730d4a14 100644 --- a/resources_servers/single_step_tool_use_with_argument_comparison/README.md +++ b/resources_servers/single_step_tool_use_with_argument_comparison/README.md @@ -8,7 +8,7 @@ Data links: ? ## Running servers The following command can be used to run this resources server, along with the tool simulation agent and an OpenAI model: ```bash -gym env run \ +gym env start \ --resources-server single_step_tool_use_with_argument_comparison \ --model-type openai_model ``` diff --git a/resources_servers/speed_bench/README.md b/resources_servers/speed_bench/README.md index defe759d25..11a233986b 100644 --- a/resources_servers/speed_bench/README.md +++ b/resources_servers/speed_bench/README.md @@ -107,7 +107,7 @@ For an EAGLE3 / MTP setup with a paired draft model, see ```bash # Running servers — uses the demo local_vllm_model config above (drop in # your own model config to swap targets; just keep the speculative_config block). -gym env run \ +gym env start \ --model-type local_vllm_model/Qwen/Qwen3-30B-A3B-Instruct-2507-ngram-specdec \ --resources-server speed_bench \ --config responses_api_agents/speed_bench_agent/configs/speed_bench_agent.yaml \ diff --git a/resources_servers/structeval/README.md b/resources_servers/structeval/README.md index 9f74e76c1b..41c67db1e8 100644 --- a/resources_servers/structeval/README.md +++ b/resources_servers/structeval/README.md @@ -28,7 +28,7 @@ reward = 0.2 * render_score + 0.8 * key_validation_score ### Running servers ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server structeval/structeval_nonrenderable ``` diff --git a/resources_servers/structured_outputs/README.md b/resources_servers/structured_outputs/README.md index 24e78ebb85..3005dc8d03 100644 --- a/resources_servers/structured_outputs/README.md +++ b/resources_servers/structured_outputs/README.md @@ -55,7 +55,7 @@ text-output and tool-call variants is in The following command runs the text-output JSON config with the simple agent and an OpenAI model: ```bash -gym env run \ +gym env start \ --model-type openai_model \ --resources-server structured_outputs/structured_outputs_json ``` @@ -74,7 +74,7 @@ For v4 tool-call structured outputs, use `structured_outputs_v4.yaml` and the v4 simple agent. The config routes through a non-executing agent because the emitted function call is the final answer, not an action to execute: ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server structured_outputs/structured_outputs_v4 ``` diff --git a/resources_servers/swerl_gen/README.md b/resources_servers/swerl_gen/README.md index bedc16245a..78f7b6a13b 100644 --- a/resources_servers/swerl_gen/README.md +++ b/resources_servers/swerl_gen/README.md @@ -60,7 +60,7 @@ You need to have singularity installed. Image having singularity: `/lustre/fsw/portfolios/llmservice/users/asohrabizade/codegen/sqsh/nvidian+nemo+verl_v2_enroot0.8.5.sqsh` ```bash -gym env run \ +gym env start \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type openai_model \ --resources-server swerl_gen diff --git a/resources_servers/swerl_llm_judge/README.md b/resources_servers/swerl_llm_judge/README.md index 0799fe1182..da57ecea25 100644 --- a/resources_servers/swerl_llm_judge/README.md +++ b/resources_servers/swerl_llm_judge/README.md @@ -55,7 +55,7 @@ Notes: **Standard format (with `grading_mode`):** ```bash -gym env run \ +gym env start \ --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ --model-type openai_model \ --resources-server swerl_llm_judge diff --git a/resources_servers/tavily_search/README.md b/resources_servers/tavily_search/README.md index 5e92d5a2bf..47fd6b3b05 100644 --- a/resources_servers/tavily_search/README.md +++ b/resources_servers/tavily_search/README.md @@ -39,7 +39,7 @@ gym dataset download --storage gitlab \ --artifact sft_samples_validation.jsonl \ --output resources_servers/tavily_search/data/sft_samples/sft_samples_validation.jsonl -gym env run \ +gym env start \ --resources-server tavily_search/tavily_search_judge_vllm_model \ --model-type vllm_model ``` diff --git a/resources_servers/terminus_judge/README.md b/resources_servers/terminus_judge/README.md index 019c082f00..808695adfe 100644 --- a/resources_servers/terminus_judge/README.md +++ b/resources_servers/terminus_judge/README.md @@ -32,7 +32,7 @@ For each verification request, the agent's JSON output is validated through mult The following command can be used to run this resources server, along with the simple agent and a policy model: ```bash -gym env run \ +gym env start \ --resources-server terminus_judge \ --model-type openai_model \ +terminus_judge_resources_server.resources_servers.terminus_judge.judge_responses_create_params.max_output_tokens=512 diff --git a/resources_servers/terminus_judge/scripts/README.md b/resources_servers/terminus_judge/scripts/README.md index 63bb28a7e8..c3547f0dbf 100644 --- a/resources_servers/terminus_judge/scripts/README.md +++ b/resources_servers/terminus_judge/scripts/README.md @@ -12,7 +12,7 @@ This pipeline uses the public `open-thoughts/OpenThoughts-Agent-v1-SFT` dataset - `datasets` - `openapi-schema-validator` 2. For smoke stage: - - `gym env run`, `gym env status`, `gym eval run --no-serve` on `PATH` + - `gym env start`, `gym env status`, `gym eval run --no-serve` on `PATH` - reachable policy/model endpoint 3. If HF cache is read-only: - `HF_HOME=/tmp/hf_home` diff --git a/resources_servers/text_to_sql/README.md b/resources_servers/text_to_sql/README.md index 91be981082..6ca3c4af1e 100644 --- a/resources_servers/text_to_sql/README.md +++ b/resources_servers/text_to_sql/README.md @@ -68,7 +68,7 @@ Each data sample should include: ### Running Servers ```bash -gym env run \ +gym env start \ --resources-server text_to_sql \ --model-type openai_model \ +text_to_sql_resources_server.resources_servers.text_to_sql.judge_responses_create_params.max_output_tokens=512 diff --git a/resources_servers/ugphysics_judge/README.md b/resources_servers/ugphysics_judge/README.md index bfeba8dfa6..7aca023e65 100644 --- a/resources_servers/ugphysics_judge/README.md +++ b/resources_servers/ugphysics_judge/README.md @@ -58,7 +58,7 @@ swap it for any other `responses_api_models/*` config that exposes a ```bash # Running servers -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server ugphysics_judge \ --config benchmarks/ugphysics/judge_gptoss20b.yaml diff --git a/resources_servers/verifif/README.md b/resources_servers/verifif/README.md index 9e67aefb97..918e7d8c0f 100644 --- a/resources_servers/verifif/README.md +++ b/resources_servers/verifif/README.md @@ -31,7 +31,7 @@ policy_model_name: gpt-5-2025-08-07 # or gpt-4.1-2025-04-14 ```bash cd /path/to/Gym source .venv/bin/activate -gym env run \ +gym env start \ --resources-server verifif \ --model-type openai_model ``` @@ -245,7 +245,7 @@ For high-throughput training: View server logs: ```bash -# Check terminal output from gym env run +# Check terminal output from gym env start # Or view Ray dashboard at http://127.0.0.1:8265 ``` diff --git a/resources_servers/vlm_eval_kit/README.md b/resources_servers/vlm_eval_kit/README.md index a18f9539ef..9b760edd23 100644 --- a/resources_servers/vlm_eval_kit/README.md +++ b/resources_servers/vlm_eval_kit/README.md @@ -54,7 +54,7 @@ python run.py --verbose \ ### Prepare data First run the VLMEvalKit server to install dependencies. ```bash -gym env run \ +gym env start \ --resources-server vlm_eval_kit \ --model-type openai_model ``` diff --git a/resources_servers/wmt_translation/README.md b/resources_servers/wmt_translation/README.md index f343e2b5e3..a99c31cc37 100644 --- a/resources_servers/wmt_translation/README.md +++ b/resources_servers/wmt_translation/README.md @@ -70,7 +70,7 @@ For an end-to-end SLURM run with COMET enabled, see the ```bash # Running servers (BLEU-only locally; flip compute_comet=true on cluster) -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server wmt_translation \ ++wmt_translation.resources_servers.wmt_translation.compute_comet=false diff --git a/resources_servers/workplace_assistant/README.md b/resources_servers/workplace_assistant/README.md index 6a9c21bd65..11c4afb8e4 100644 --- a/resources_servers/workplace_assistant/README.md +++ b/resources_servers/workplace_assistant/README.md @@ -11,7 +11,7 @@ Commands - Spin up server: ``` -gym env run \ +gym env start \ --model-type openai_model \ --resources-server workplace_assistant ``` diff --git a/resources_servers/xlam_fc/README.md b/resources_servers/xlam_fc/README.md index 8079381a96..475abfd0bc 100644 --- a/resources_servers/xlam_fc/README.md +++ b/resources_servers/xlam_fc/README.md @@ -9,7 +9,7 @@ python resources_servers/xlam_fc/generate_dataset.py ``` ```bash -gym env run \ +gym env start \ --model-type vllm_model \ --resources-server xlam_fc ``` diff --git a/resources_servers/xstest/README.md b/resources_servers/xstest/README.md index 82d46c0b15..80ef2af9a6 100644 --- a/resources_servers/xstest/README.md +++ b/resources_servers/xstest/README.md @@ -100,12 +100,12 @@ contrast_privacy ### Example usage ```bash # For chat completions endpoints (vLLM, NIM, etc.): -gym env run \ +gym env start \ --resources-server xstest \ --model-type vllm_model # For OpenAI Responses API endpoints: -# gym env run --resources-server xstest --model-type openai_model +# gym env start --resources-server xstest --model-type openai_model gym eval run --no-serve \ --agent xstest_simple_agent \ diff --git a/responses_api_agents/claude_code_agent/README.md b/responses_api_agents/claude_code_agent/README.md index 0f61020d10..878ee9e2ed 100644 --- a/responses_api_agents/claude_code_agent/README.md +++ b/responses_api_agents/claude_code_agent/README.md @@ -29,7 +29,7 @@ anthropic_base_url: http://localhost:8000 No model server is needed for basic eval. To extend this agent to training, a model server should be developed that handles messages endpoint. For evals with the current version, just pass the resources server config, which includes the agent server config, as is the current standard in NeMo Gym: ```bash -gym env run --resources-server reasoning_gym/reasoning_gym_claude_code_agent +gym env start --resources-server reasoning_gym/reasoning_gym_claude_code_agent ``` ### Run the agent diff --git a/responses_api_agents/harbor_agent/README.md b/responses_api_agents/harbor_agent/README.md index 29789759b9..223b10b325 100644 --- a/responses_api_agents/harbor_agent/README.md +++ b/responses_api_agents/harbor_agent/README.md @@ -200,7 +200,7 @@ export APPTAINER_DOCKER_PASSWORD= Then start NeMo Gym: ```bash -gym env run \ +gym env start \ --config responses_api_agents/harbor_agent/configs/harbor_agent.yaml \ --model-type vllm_model/vllm_model_for_training ``` @@ -289,7 +289,7 @@ policy_model_name: Then follow the same Harbor-agent workflow with the Daytona config: ```bash -gym env run \ +gym env start \ --config responses_api_agents/harbor_agent/configs/harbor_agent_daytona.yaml \ --model-type vllm_model ``` @@ -297,7 +297,7 @@ gym env run \ Alternatively, pass those values as CLI overrides: ```bash -gym env run \ +gym env start \ --config responses_api_agents/harbor_agent/configs/harbor_agent_daytona.yaml \ --model-type vllm_model \ --model-url \ diff --git a/responses_api_agents/hermes_agent/README.md b/responses_api_agents/hermes_agent/README.md index a136108312..951b3bfa72 100644 --- a/responses_api_agents/hermes_agent/README.md +++ b/responses_api_agents/hermes_agent/README.md @@ -13,7 +13,7 @@ policy_model_name: gpt-4o ## Launch nemo gym servers ```bash -gym env run \ +gym env start \ --resources-server math_with_judge/math_with_judge_hermes_agent \ --model-type openai_model ``` diff --git a/responses_api_agents/langgraph_agent/README.md b/responses_api_agents/langgraph_agent/README.md index c9a229b7da..6b256d9619 100644 --- a/responses_api_agents/langgraph_agent/README.md +++ b/responses_api_agents/langgraph_agent/README.md @@ -9,7 +9,7 @@ Please note that agents such as parallel thinking which produce non-monotonicall ## Quick Start ```bash -gym env run \ +gym env start \ --resources-server reasoning_gym/reflection_agent \ --model-type vllm_model ``` diff --git a/responses_api_agents/mini_swe_agent/README.md b/responses_api_agents/mini_swe_agent/README.md index 823def449a..ab5e12baaa 100644 --- a/responses_api_agents/mini_swe_agent/README.md +++ b/responses_api_agents/mini_swe_agent/README.md @@ -107,7 +107,7 @@ gym dataset download --storage gitlab \ --output data/train.jsonl # Start server -gym env run \ +gym env start \ --resources-server mini_swe_agent \ --model-type openai_model & \ '+mini_swe_simple_agent.responses_api_agents.mini_swe_agent.cache_dir_template=/path/to/images/xingyaoww_sweb.eval.x86_64.\{instance_id\}.sif' \ diff --git a/responses_api_agents/stirrup_agent/README.md b/responses_api_agents/stirrup_agent/README.md index c07ddd7ba5..2de472890f 100644 --- a/responses_api_agents/stirrup_agent/README.md +++ b/responses_api_agents/stirrup_agent/README.md @@ -174,7 +174,7 @@ apptainer build gdpval.sif responses_api_agents/stirrup_agent/containers/gdpval. Then: ```yaml -# env.yaml or gym env run override +# env.yaml or gym env start override stirrup_agent: responses_api_agents: stirrup_agent: diff --git a/responses_api_agents/swe_agents/README.md b/responses_api_agents/swe_agents/README.md index b041564a94..d7e4b4c668 100644 --- a/responses_api_agents/swe_agents/README.md +++ b/responses_api_agents/swe_agents/README.md @@ -254,7 +254,7 @@ policy_model_name: Qwen/Qwen3-Coder-30B-A3B-Instruct # OpenHands single-prompt (swap to # responses_api_agents/swe_agents/configs/swebench_multi_tools.yaml for full # prompt × agent-class × tool-name diversity) -gym env run \ +gym env start \ --config responses_api_agents/swe_agents/configs/swebench_openhands.yaml \ --model-type vllm_model \ +swe_agents.responses_api_agents.swe_agents.container_formatter=/lustre/xxx/images/swe-bench/swebench_sweb.eval.x86_64.\{instance_id\}.sif \ diff --git a/responses_api_agents/tau2/README.md b/responses_api_agents/tau2/README.md index 605dcc1ba1..8a8426e03f 100644 --- a/responses_api_agents/tau2/README.md +++ b/responses_api_agents/tau2/README.md @@ -1,6 +1,6 @@ # Description ```bash -gym env run \ +gym env start \ --benchmark tau2 \ --model-type openai_model \ ++nemo_gym_log_dir=results/tau2 \ diff --git a/responses_api_agents/verifiers_agent/README.md b/responses_api_agents/verifiers_agent/README.md index 017edc1684..4502949311 100644 --- a/responses_api_agents/verifiers_agent/README.md +++ b/responses_api_agents/verifiers_agent/README.md @@ -21,7 +21,7 @@ policy_model_name: "Qwen/Qwen3-4B-Instruct-2507" ``` # start nemo gym servers -gym env run \ +gym env start \ --config responses_api_agents/verifiers_agent/configs/acereason-math.yaml \ --model-type vllm_model @@ -100,7 +100,7 @@ deactivate source .venv/bin/activate # start nemo gym servers -gym env run \ +gym env start \ --config responses_api_agents/verifiers_agent/configs/ascii-tree.yaml \ --model-type vllm_model @@ -116,7 +116,7 @@ gym eval run --no-serve \ The patch to include prompt and generation token ids for preventing retokenization error when training with NeMo RL has been upstreamed into verifiers' `NeMoRLChatCompletionsClient`. We currently track verifiers `main` (`git+https://github.com/PrimeIntellect-ai/verifiers.git@main`) for this support; once verifiers `0.1.13` is released, the requirements pin will move to that tagged version. -For installing new prime environments and generating datasets, use a separate venv (outside of Gym) to avoid dependency conflicts with the `exclude-dependencies` section of Gym `pyproject.toml` and the server's pinned verifiers version. After generating your dataset, deactivate the separate venv and return to the Gym venv for running servers. Make sure to restart NeMo Gym servers with `gym env run` after any environment changes to ensure the pinned version of verifiers is used. +For installing new prime environments and generating datasets, use a separate venv (outside of Gym) to avoid dependency conflicts with the `exclude-dependencies` section of Gym `pyproject.toml` and the server's pinned verifiers version. After generating your dataset, deactivate the separate venv and return to the Gym venv for running servers. Make sure to restart NeMo Gym servers with `gym env start` after any environment changes to ensure the pinned version of verifiers is used. # Licensing information Code: Apache 2.0 diff --git a/responses_api_models/azure_openai_model/README.md b/responses_api_models/azure_openai_model/README.md index d0e9ffedda..8be7b42a44 100644 --- a/responses_api_models/azure_openai_model/README.md +++ b/responses_api_models/azure_openai_model/README.md @@ -15,7 +15,7 @@ policy_model_name: gpt-5-nano ### Running the server Set the API version. It usually looks something like "2024-10-21". ```bash -gym env run \ +gym env start \ --model-type azure_openai_model \ --resources-server equivalence_llm_judge \ +policy_model.responses_api_models.azure_openai_model.default_query.api-version= diff --git a/responses_api_models/local_vllm_model/README.md b/responses_api_models/local_vllm_model/README.md index 78429dede7..d65ea81dd4 100644 --- a/responses_api_models/local_vllm_model/README.md +++ b/responses_api_models/local_vllm_model/README.md @@ -5,7 +5,7 @@ Run this on a single GPU node! Set tensor_parallel_size * data_parallel_size to the number of GPUs on your node. For this single node config, data_parallel_size_local is equal to data_parallel_size ```bash -gym env run \ +gym env start \ --resources-server example_single_tool_call \ --config responses_api_models/local_vllm_model/configs/nano_v3_single_node.yaml &> temp.log & \ ++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.tensor_parallel_size=4 \ diff --git a/responses_api_models/local_vllm_model_proxy/README.md b/responses_api_models/local_vllm_model_proxy/README.md index fa87d6a548..df835d45b5 100644 --- a/responses_api_models/local_vllm_model_proxy/README.md +++ b/responses_api_models/local_vllm_model_proxy/README.md @@ -3,7 +3,7 @@ # End-to-end test with GPT OSS 20B reasoning high ```bash -gym env run \ +gym env start \ --model-type local_vllm_model/openai/gpt-oss-20b-reasoning-high \ --model-type local_vllm_model_proxy \ ++policy_model_proxy.responses_api_models.local_vllm_model_proxy.model_server.name=gpt-oss-20b-reasoning-high \ diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 69659155e1..08434803bb 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -51,7 +51,7 @@ def _split_overrides(overrides: list[str]) -> tuple[set[str], set[str]]: # `gym ` -> the legacy ng_ function it dispatches to, for the config-accepting commands. CONFIG_COMMANDS = [ - (["env", "run"], "nemo_gym.cli.env:run"), + (["env", "start"], "nemo_gym.cli.env:run"), (["env", "resolve"], "nemo_gym.cli.env:dump_config"), (["eval", "prepare"], "nemo_gym.cli.eval:prepare_benchmark"), (["eval", "aggregate"], "nemo_gym.cli.eval:aggregate_rollouts"), @@ -69,7 +69,7 @@ def test_config_becomes_config_paths(self, monkeypatch: MonkeyPatch, command, ex assert overrides == ["+config_paths=[my.yaml]"] def test_repeated_config_joined_into_one_list(self, monkeypatch: MonkeyPatch) -> None: - _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--config", "a.yaml", "--config", "b.yaml"]) + _, overrides = _dispatch_for(monkeypatch, ["env", "start", "--config", "a.yaml", "--config", "b.yaml"]) # We have this set of asserts to avoid asserting configs order in the string assert len(overrides) == 1 @@ -80,13 +80,13 @@ def test_repeated_config_joined_into_one_list(self, monkeypatch: MonkeyPatch) -> assert "b.yaml" in override def test_config_is_prepended_before_passthrough_overrides(self, monkeypatch: MonkeyPatch) -> None: - _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--config", "a.yaml", "+foo=bar"]) + _, overrides = _dispatch_for(monkeypatch, ["env", "start", "--config", "a.yaml", "+foo=bar"]) assert len(overrides) == 2 assert "+config_paths=[a.yaml]" in overrides assert "+foo=bar" in overrides def test_without_config_no_config_paths_added(self, monkeypatch: MonkeyPatch) -> None: - _, overrides = _dispatch_for(monkeypatch, ["env", "run", "+foo=bar"]) + _, overrides = _dispatch_for(monkeypatch, ["env", "start", "+foo=bar"]) assert overrides == ["+foo=bar"] def test_config_rejected_on_non_config_command(self, monkeypatch: MonkeyPatch) -> None: @@ -446,7 +446,7 @@ def test_model_flags(self, monkeypatch: MonkeyPatch) -> None: monkeypatch, [ "env", - "run", + "start", "--config", "c.yaml", "--model", @@ -480,7 +480,7 @@ def test_local_vllm_deployment_flow(self, monkeypatch: MonkeyPatch) -> None: assert others == {"+policy_model_name=Qwen/Qwen3-8B"} def test_short_alias_on_env_run(self, monkeypatch: MonkeyPatch) -> None: - _, overrides = _dispatch_for(monkeypatch, ["env", "run", "-m", "/ckpt/path"]) + _, overrides = _dispatch_for(monkeypatch, ["env", "start", "-m", "/ckpt/path"]) assert overrides == ["+policy_model_name=/ckpt/path"] @@ -606,21 +606,24 @@ class TestAssetSelectors: # benchmarks/aime25-x/README.md: ng_prepare_benchmark "+config_paths=[benchmarks/aime25-x/config.yaml]" (["eval", "prepare", "--benchmark", "aime25-x"], "benchmarks/aime25-x/config.yaml"), # benchmarks/gpqa/README.md: ng_run "+config_paths=[benchmarks/gpqa/config.yaml]" (start a benchmark's servers) - (["env", "run", "--benchmark", "gpqa"], "benchmarks/gpqa/config.yaml"), + (["env", "start", "--benchmark", "gpqa"], "benchmarks/gpqa/config.yaml"), # README.md / quickstart.mdx: resources_servers/mcqa/configs/mcqa.yaml - (["env", "run", "--resources-server", "mcqa"], "resources_servers/mcqa/configs/mcqa.yaml"), + (["env", "start", "--resources-server", "mcqa"], "resources_servers/mcqa/configs/mcqa.yaml"), # model-server/vllm.mdx: resources_servers/example_multi_step/configs/example_multi_step.yaml ( - ["env", "run", "--resources-server", "example_multi_step"], + ["env", "start", "--resources-server", "example_multi_step"], "resources_servers/example_multi_step/configs/example_multi_step.yaml", ), # README.md / quickstart.mdx: responses_api_models/openai_model/configs/openai_model.yaml ( - ["env", "run", "--model-type", "openai_model"], + ["env", "start", "--model-type", "openai_model"], "responses_api_models/openai_model/configs/openai_model.yaml", ), # model-server/vllm.mdx: responses_api_models/vllm_model/configs/vllm_model.yaml - (["env", "run", "--model-type", "vllm_model"], "responses_api_models/vllm_model/configs/vllm_model.yaml"), + ( + ["env", "start", "--model-type", "vllm_model"], + "responses_api_models/vllm_model/configs/vllm_model.yaml", + ), ], ) def test_name_resolves_to_config_path(self, monkeypatch: MonkeyPatch, argv, expected_config) -> None: @@ -632,7 +635,7 @@ def test_quickstart_resource_server_plus_model(self, monkeypatch: MonkeyPatch) - # ng_run "+config_paths=[resources_servers/mcqa/configs/mcqa.yaml, # responses_api_models/openai_model/configs/openai_model.yaml]" target, overrides = _dispatch_for( - monkeypatch, ["env", "run", "--resources-server", "mcqa", "--model-type", "openai_model"] + monkeypatch, ["env", "start", "--resources-server", "mcqa", "--model-type", "openai_model"] ) assert target == "nemo_gym.cli.env:run" paths, others = _split_overrides(overrides) @@ -728,8 +731,8 @@ def test_benchmark_flavor_syntax(self, monkeypatch: MonkeyPatch) -> None: def test_environment_selector_resolves_to_config(self, monkeypatch: MonkeyPatch) -> None: # Environments mirror benchmarks: `--environment ` loads `environments//config.yaml`. - # Used by `gym env run` / `gym eval run` for environments/ (see environments/*/README.md). - _, overrides = _dispatch_for(monkeypatch, ["env", "run", "--environment", "circle_count"]) + # Used by `gym env start` / `gym eval run` for environments/ (see environments/*/README.md). + _, overrides = _dispatch_for(monkeypatch, ["env", "start", "--environment", "circle_count"]) assert overrides == [f"+config_paths=[{WORKING_DIR / 'environments/circle_count/config.yaml'}]"] def test_selectors_merge_into_single_config_paths(self, monkeypatch: MonkeyPatch) -> None: @@ -760,7 +763,7 @@ def test_unknown_benchmark_errors_with_available_hint(self, monkeypatch: MonkeyP def test_unknown_flavor_error_points_at_configs_dir(self, monkeypatch: MonkeyPatch, capsys) -> None: # For a known server with an unknown flavor, the hint should point at that server's configs/ dir. monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) - monkeypatch.setattr(sys, "argv", ["gym", "env", "run", "--resources-server", "mcqa/nope"]) + monkeypatch.setattr(sys, "argv", ["gym", "env", "start", "--resources-server", "mcqa/nope"]) with pytest.raises(SystemExit): main() err = capsys.readouterr().err diff --git a/tests/unit_tests/test_server_status.py b/tests/unit_tests/test_server_status.py index 3157afad8a..2a17e8cad1 100644 --- a/tests/unit_tests/test_server_status.py +++ b/tests/unit_tests/test_server_status.py @@ -227,4 +227,4 @@ def test_discover_servers_head_server_down(self, monkeypatch: MonkeyPatch, capsy # The warning goes through logging (stderr), keeping stdout machine-readable for `--json`. assert capsys.readouterr().out == "" assert "Could not connect to head server" in caplog.text - assert "gym env run" in caplog.text + assert "gym env start" in caplog.text From 534ea801ae81a5700b28d3c927a9ee41e5285da1 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Wed, 24 Jun 2026 15:03:36 +0200 Subject: [PATCH 34/40] chore: polish docs Signed-off-by: Marta Stepniewska-Dziubinska --- .../latest/pages/reference/cli-commands.mdx | 103 ++++++++++-------- 1 file changed, 58 insertions(+), 45 deletions(-) diff --git a/fern/versions/latest/pages/reference/cli-commands.mdx b/fern/versions/latest/pages/reference/cli-commands.mdx index 76a44cf01c..9196081b7a 100644 --- a/fern/versions/latest/pages/reference/cli-commands.mdx +++ b/fern/versions/latest/pages/reference/cli-commands.mdx @@ -68,11 +68,24 @@ These options are shared across many commands. The `--benchmark`, `--resources-server`, and `--model-type` selectors resolve a component name to its config file for you, so you do not need to know the project's directory layout. If you mistype a name, the CLI suggests the closest match. To point at a config file directly, use `--config ` instead. + +### Selecting the model server + +Commands that need a model (`gym env start`, `gym eval run`) configure it through four flags: + +| Flag | Description | +| --- | --- | +| `--model-type NAME` | The model server type to load (such as `openai_model`, `vllm_model`, or `local_vllm_model`). | +| `--model`, `-m` | The served model identifier: an API model name, an HF id, or a local checkpoint path. Interpreted per `--model-type`. Maps to `policy_model_name`. | +| `--model-url` | Base URL of an existing model server endpoint. Maps to `policy_base_url`. | +| `--model-api-key` | API key for the model server. Maps to `policy_api_key`. | + + ### Hydra overrides (escape hatch) -The `gym` CLI is a thin wrapper over Gym's Hydra config system. Standard flags (`--flag value`) cover the common inputs. Anything not covered by a flag can still be passed as a raw Hydra override using `+key=value` (add a new key) or `++key=value` (override an existing key). Unknown overrides are forwarded to Hydra untouched, so advanced config composition keeps working. +The `gym` CLI is a thin wrapper over Gym's Hydra config system. Standard flags (`--flag value`) cover the common inputs. Anything not covered by a flag can still be passed as a raw Hydra override using `+key=value` (add a new key) or `++key=value` (add or override an existing key). Unknown overrides are forwarded to Hydra untouched, so advanced config composition is fully functional. -When a command needs overrides for keys that have no dedicated flag, prefer keeping the whole command in Hydra form (config paths included) rather than mixing `--flag` and `+key=value` styles in one invocation: +When a command needs overrides for keys that have no dedicated flag, you can keep the whole command in Hydra form (config paths included) or mix `--flag` and `+key=value` styles in one invocation: ```bash gym eval run \ @@ -82,26 +95,13 @@ gym eval run \ +wandb_project=gym-dev ``` -Unknown `--flags` (note the leading dashes) are treated as typos and rejected with a "did you mean?" hint, since Hydra overrides never start with `-`. - -### Selecting the model server - -Commands that need a model (`gym env start`, `gym eval run`) configure it through four flags: - -| Flag | Description | -| --- | --- | -| `--model-type NAME` | The model server type to load (such as `openai_model`, `vllm_model`, or `local_vllm_model`). Derives the correct server implementation. | -| `--model`, `-m` | The served model identifier: an API model name, an HF id, or a local checkpoint path. Interpreted per `--model-type`. Maps to `policy_model_name`. | -| `--model-url` | Base URL of an existing model server endpoint. Maps to `policy_base_url`. | -| `--model-api-key` | API key for the model server. Maps to `policy_api_key`. | - --- ## General ### `gym --help` -List all command groups. Replaces `ng_help`. +List all command groups and their descriptions. ```bash gym --help @@ -109,7 +109,7 @@ gym --help ### `gym --version` -Print the NeMo Gym version along with Python, key dependency, and system information. Replaces `ng_version`. +Print the NeMo Gym version along with Python, key dependency, and system information. | Option | Description | | --- | --- | @@ -145,11 +145,11 @@ gym list benchmarks --json | jq '.[].name' ### `gym search` -List benchmarks filtered to those whose name, agent, resources server, dataset, or domain fuzzily matches a query. Equivalent to `gym list benchmarks` narrowed to a query. +List benchmarks whose name, agent, resources server, dataset, or domain matches a query. | Argument / Option | Description | | --- | --- | -| `QUERY` | Substring to match against component names (positional, required). | +| `QUERY` | Query to match against component names (positional, required). | | `--json` | Output matches as JSON. | ```bash @@ -161,11 +161,11 @@ gym search aime --json ## Datasets -Commands for preparing, previewing, and managing datasets. By default dataset transfer commands use HuggingFace; pass `--storage gitlab` to target the GitLab Model Registry. +Commands for preparing, previewing, and managing datasets. By default dataset transfer commands use HuggingFace; pass `--storage gitlab` to target the GitLab Registry. ### `gym dataset upload` -Upload a prepared local JSONL dataset to HuggingFace (default) or GitLab. Replaces `ng_upload_dataset_to_hf` and `ng_upload_dataset_to_gitlab`. +Upload a prepared local JSONL dataset to HuggingFace (default) or GitLab. | Option | Description | | --- | --- | @@ -174,7 +174,7 @@ Upload a prepared local JSONL dataset to HuggingFace (default) or GitLab. Replac | `--name` | Dataset name. | | `--revision` | Dataset revision (version). | | `--split` | Dataset split (HF only). | -| `--create-pr` | Open a pull request instead of committing directly (HF only). | +| `--create-pr` | Open a pull request with your changes (HF only). | ```bash # Upload to HuggingFace @@ -193,7 +193,7 @@ gym dataset upload \ ### `gym dataset download` -Download a dataset from HuggingFace (default) or GitLab. Replaces `ng_download_dataset_from_hf` and `ng_download_dataset_from_gitlab`. +Download a dataset from HuggingFace (default) or GitLab. | Option | Description | | --- | --- | @@ -223,7 +223,7 @@ gym dataset download \ ### `gym dataset rm` -Delete a dataset from the GitLab Model Registry. Prompts for confirmation. Replaces `ng_delete_dataset_from_gitlab`. +Delete a dataset from the GitLab Registry. Prompts for confirmation. | Option | Description | | --- | --- | @@ -235,7 +235,7 @@ gym dataset rm --name old_dataset ### `gym dataset migrate` -Upload a local JSONL dataset to HuggingFace and delete it from GitLab afterward. Use `gym dataset upload` if you want optional rather than automatic GitLab deletion. Replaces `ng_gitlab_to_hf_dataset`. +Migrate a JSONL dataset to HuggingFace from GitLab. Use `gym dataset upload` if you do not want automatic GitLab deletion. | Option | Description | | --- | --- | @@ -243,7 +243,7 @@ Upload a local JSONL dataset to HuggingFace and delete it from GitLab afterward. | `--name` | Dataset name. | | `--revision` | Dataset revision (HF). | | `--split` | Dataset split. | -| `--create-pr` | Open a pull request instead of committing directly. | +| `--create-pr` | Open a pull request to HF dataset with your changes. | ```bash gym dataset migrate \ @@ -254,7 +254,7 @@ gym dataset migrate \ ### `gym dataset render` -Generate a dataset preview by materializing prompts from a raw input file and a prompt template. Replaces `ng_materialize_prompts`. +Generate a dataset preview by materializing prompts from a raw input file and a prompt template. | Option | Description | | --- | --- | @@ -271,21 +271,20 @@ gym dataset render \ ### `gym dataset collate` -Validate and collate a dataset, generating metrics and statistics. Replaces `ng_prepare_data`. +Validate and collate a dataset, generating metrics and statistics. | Option | Description | | --- | --- | | `--config PATH` | Config file to load. Repeatable. | | `--resources-server NAME` | Load the named resources server config. | | `--search-dir DIR` | Extra root directory to search for named components. Repeatable. | -| `--mode {train_preparation,example_validation}` | Use `train_preparation` to prepare train/validation datasets, or `example_validation` to validate example data for PR submission. | +| `--mode {train_preparation,example_validation}` | Use `train_preparation` to prepare train/validation datasets, or `example_validation` to validate example data. | | `--output-dir` | Output directory for the prepared data. | | `--download` | Download source datasets before collating. | ```bash gym dataset collate \ --resources-server example_multi_step \ - --config responses_api_models/openai_model/configs/openai_model.yaml \ --output-dir data/example_multi_step \ --mode example_validation ``` @@ -298,7 +297,7 @@ Commands for developing, running, and inspecting environments (a dataset + agent ### `gym env init` -Scaffold a new resources server with template files (config, app, tests, README, data directory). Replaces `ng_init_resources_server`. +Scaffold a new resources server with template files (config, app, tests, README, data directory). | Option | Description | | --- | --- | @@ -310,19 +309,23 @@ gym env init --resources-server my_server ### `gym env resolve` -Resolve the configs, flags, and overrides into a final merged config and print it. Useful for debugging configuration. Secrets are hidden. Replaces `ng_dump_config`. +Resolve the configs, flags, and overrides into a final merged config and print it. Useful for debugging configuration. Secrets are hidden. | Option | Description | | --- | --- | | `--config PATH` | Config file to load. Repeatable. | ```bash -gym env resolve --config resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml +# Merge a resources-server config with a model config, apply an override, and print the result +gym env resolve \ + --config resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + ++responses_create_params.temperature=0.6 ``` ### `gym env packages` -Each server has its own isolated virtual environment. List the packages installed in a server's environment. Replaces `ng_pip_list`. +Each server has its own isolated virtual environment. List the packages installed in a server's environment. | Option | Description | | --- | --- | @@ -341,7 +344,7 @@ gym env packages \ ### `gym env test` -Test resource server(s) by running their pytest suite. If no resources server is given, all of them are tested. Replaces `ng_test` and `ng_test_all`. +Test resource server(s) by running their pytest suite. If no resources server is given, all of them are tested. | Option | Description | | --- | --- | @@ -357,7 +360,7 @@ gym env test ### `gym env start` -Start the NeMo Gym servers (agents, models, resources) defined by the provided configs. Reads configuration from YAML files and runs each configured server in its own environment. Replaces `ng_run`. +Start the NeMo Gym servers (agents, models, resources) defined by the provided configs. Reads configuration from YAML files and runs each configured server in its own environment. | Option | Description | | --- | --- | @@ -383,7 +386,7 @@ gym env start \ ### `gym env status` -Show all currently running NeMo Gym servers and their health. Replaces `ng_status`. +Show all currently running NeMo Gym servers and their health. | Option | Description | | --- | --- | @@ -413,11 +416,11 @@ NeMo Gym Server Status: ## Evaluation -Commands for running evaluations end to end: prepare data, collect rollouts, aggregate, and profile. +Commands for running evaluations end to end: prepare data, collect rollouts, aggregate for sharded runs, and profile. ### `gym eval prepare` -Prepare a benchmark's data by running its `prepare.py` script and dump the result to disk. Replaces `ng_prepare_benchmark`. +Prepare a benchmark's data by running its `prepare.py` script and dump the result to disk. | Option | Description | | --- | --- | @@ -431,7 +434,7 @@ gym eval prepare --benchmark aime24 ### `gym eval run` -Collate data, start the servers, and collect rollouts. This is the main evaluation command. By default it spins up all required servers (the former `ng_e2e_collect_rollouts`). Pass `--no-serve` to collect against servers you already started with `gym env start` (the former `ng_collect_rollouts`). Replaces `ng_e2e_collect_rollouts` and `ng_collect_rollouts`. +Collate data, start the servers, and collect rollouts. This is the main evaluation command. By default it spins up all required servers. Pass `--no-serve` to collect against servers you already started with `gym env start`. | Option | Description | | --- | --- | @@ -480,7 +483,7 @@ gym eval run --no-serve \ ### `gym eval aggregate` -Merge sharded rollout results into a single rollouts file with aggregate metrics. Replaces `ng_aggregate_rollouts`. +Merge sharded rollout results into a single rollouts file with aggregate metrics. | Option | Description | | --- | --- | @@ -493,7 +496,7 @@ gym eval aggregate --output results/merged.jsonl ### `gym eval profile` -Compute a reward profile from collected rollouts. Outputs per-task statistics such as average reward, standard deviation, min/max, and pass rate, useful for filtering tasks before training by difficulty or variance. Requires rollouts collected with `--num-repeats` greater than 1. Replaces `ng_reward_profile`. +Compute a reward profile from collected rollouts. Outputs per-task statistics such as average reward, standard deviation, min/max, and pass rate, useful for filtering tasks before training by difficulty or variance. Requires rollouts collected with `--num-repeats` greater than 1. | Option | Description | | --- | --- | @@ -512,7 +515,7 @@ gym eval profile \ ### `gym dev test` -Run NeMo Gym's core unit tests with coverage reporting. Replaces `ng_dev_test`. +Run NeMo Gym's core unit tests with coverage reporting. ```bash gym dev test @@ -522,7 +525,8 @@ gym dev test ## Migrating from the legacy commands -The legacy `ng_*` and `nemo_gym_*` console scripts are deprecated but still functional. Each prints a one-time deprecation notice that names its `gym` replacement, then forwards your arguments (including any Hydra overrides) to the new command. Scripts, CI pipelines, and muscle memory keep working during the deprecation period; update them to the `gym` syntax when convenient. +The legacy `ng_*` and `nemo_gym_*` are deprecated and will be removed in the future release. +Use the tables below to find their `gym` replacement and update your scripts and workflows. ### Command mapping @@ -556,7 +560,7 @@ The legacy `ng_*` and `nemo_gym_*` console scripts are deprecated but still func ### Replacing Hydra overrides with flags -The most common Hydra overrides now have dedicated flags. Prefer flags; fall back to `+key=value` / `++key=value` for anything without one. +The most common Hydra overrides now have dedicated flags: | Legacy Hydra override | New flag | | --- | --- | @@ -577,6 +581,15 @@ The most common Hydra overrides now have dedicated flags. Prefer flags; fall bac | `+mode=...` | `--mode` | | `+output_dirpath=...` | `--output-dir` | | `+should_download=true` | `--download` | +| `+prompt_config=...` | `--prompt-config` | +| `+resume_from_cache=true` | `--resume` | +| `+dataset_name=...` | `--name` | +| `+repo_id=...` | `--repo-id` | +| `+revision=...` / `+version=...` | `--revision` | +| `+artifact_fpath=...` | `--artifact` | +| `+output_fpath=...` | `--output`, `-o` | +| `+create_pr=true` | `--create-pr` | +| `+outdated=true` | `--outdated` | ### Common workflows, before and after From e150eddcae3487cffca99027a7ebf3a732df4128 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Thu, 25 Jun 2026 08:59:38 +0200 Subject: [PATCH 35/40] Update nemo_gym/cli/main.py Co-authored-by: Ananth Subramaniam --- nemo_gym/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index d3917f5a52..a662e25a74 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -310,7 +310,7 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: # GitLab stores it as `version`, HF as `revision`; emit both and let each backend keep its own. Flag( register=lambda p: p.add_argument( - "--revision", dest="revision", help="Dataset revision (version) to download." + "--revision", dest="revision", help="Dataset revision (version) to upload." ), translate_to_hydra=lambda args: ( # we set both version and revision because GitLab and HF use different keys From e552e3ab60c2ab73cae44f8f4b964926e26e68d0 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Thu, 25 Jun 2026 09:16:54 +0200 Subject: [PATCH 36/40] fix: add --input-glob flag to gym eval aggregate Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 6 ++++++ tests/unit_tests/test_cli_main.py | 8 +++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index a662e25a74..9342a33f42 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -467,6 +467,12 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: summary="Aggregate sharded rollout results.", flags=( CONFIG, + _value_flag( + "input-glob", + "input_glob", + "Glob (or comma-separated globs) matching the rollout shards to aggregate.", + aliases=("-i",), + ), _value_flag( "output", "output_jsonl_fpath", diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index 08434803bb..a910670e54 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -415,10 +415,12 @@ def test_collate_mode_rejects_invalid_choice(self, monkeypatch: MonkeyPatch) -> class TestEvalAggregateFlags: - def test_output_flag(self, monkeypatch: MonkeyPatch) -> None: - target, overrides = _dispatch_for(monkeypatch, ["eval", "aggregate", "-o", "out.jsonl"]) + def test_aggregate_flags(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, ["eval", "aggregate", "-i", "results/rollouts-*.jsonl", "-o", "out.jsonl"] + ) assert target == "nemo_gym.cli.eval:aggregate_rollouts" - assert overrides == ["+output_jsonl_fpath=out.jsonl"] + assert set(overrides) == {"+input_glob=results/rollouts-*.jsonl", "+output_jsonl_fpath=out.jsonl"} class TestEvalProfileFlags: From 34b57f1fdde01b2cf273d8e0ea181abc43c452b6 Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Thu, 25 Jun 2026 09:21:07 +0200 Subject: [PATCH 37/40] fix: update comment Signed-off-by: Marta Stepniewska-Dziubinska --- nemo_gym/cli/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 9342a33f42..7f99b17701 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -136,7 +136,8 @@ def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: ) # Shared flag: emit machine-readable JSON instead of human output. Reused by reporting commands (version, list, -# env status). The reserved `json` config key is read centrally via nemo_gym.cli.output.emit. +# env status). Each command reads the reserved `json` config key ad hoc via +# global_config_dict.get(JSON_OUTPUT_KEY_NAME) (see general.py, eval.py, env.py). JSON = _bool_flag("json", "json", "Output as JSON for programmatic use.") # Positional search query for `gym search`; surfaced to the listing command as the `query` config key. From 7e1525fb92a52f77dd7a77acd09aa6e2d0b8fe1b Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Thu, 25 Jun 2026 09:56:03 +0200 Subject: [PATCH 38/40] feat: add unit tests for dispatch Signed-off-by: Marta Stepniewska-Dziubinska --- tests/unit_tests/test_cli_main.py | 63 +++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py index a910670e54..de9bed828f 100644 --- a/tests/unit_tests/test_cli_main.py +++ b/tests/unit_tests/test_cli_main.py @@ -14,6 +14,7 @@ # limitations under the License. import logging import sys +import types import pytest from pytest import MonkeyPatch @@ -423,6 +424,68 @@ def test_aggregate_flags(self, monkeypatch: MonkeyPatch) -> None: assert set(overrides) == {"+input_glob=results/rollouts-*.jsonl", "+output_jsonl_fpath=out.jsonl"} +class TestDispatch: + """Exercise the real `dispatch` (every router test above stubs it), so argv rewriting and + module/attribute resolution are actually covered.""" + + def test_dispatch_imports_target_and_rewrites_argv(self, monkeypatch: MonkeyPatch) -> None: + # Direct unit test of dispatch (no parser): it splits "module:func", imports the module, + # resolves the attribute, rewrites sys.argv to argv[0] + overrides, then calls the target. + # A synthetic module registered in sys.modules keeps this decoupled from any real target. + captured: dict = {} + + def recorder() -> None: + captured["argv"] = list(sys.argv) + + fake_module = types.ModuleType("my_module.my_submodule") + fake_module.my_function = recorder + monkeypatch.setitem(sys.modules, "my_module.my_submodule", fake_module) + # argv starts with the parsed command tokens; dispatch must drop them but keep argv[0]. + monkeypatch.setattr(sys, "argv", ["my_command", "subcommand", "--some-flag", "some-value"]) + cli_main.dispatch("my_module.my_submodule:my_function", ["+a=1", "+b=2"]) + assert captured["argv"] == ["my_command", "+a=1", "+b=2"] + + def test_main_to_dispatch_end_to_end(self, monkeypatch: MonkeyPatch) -> None: + # Drive dispatch from main() with only the leaf target stubbed: the target string is split, + # the module is imported, the attribute is resolved, and sys.argv is rewritten for real. + # Mix `--flag` asset selectors (translated and coalesced into +config_paths) with raw + # `+`/`++` Hydra passthroughs. + captured: dict = {} + + def recorder() -> None: + captured["argv"] = list(sys.argv) + + monkeypatch.setattr("nemo_gym.cli.eval.e2e_rollout_collection", recorder) + monkeypatch.setattr( + sys, + "argv", + [ + "gym", + "eval", + "run", + "--benchmark", + "aime24", + "--model-type", + "openai_model", + "++responses_create_params.reasoning.effort=low", + "+wandb_project=gym-dev", + ], + ) + main() + # dispatch drops the parsed command tokens, keeping argv[0] and the resolved overrides. + assert captured["argv"][0] == "gym" + config_paths, others = _split_overrides(captured["argv"][1:]) + assert any(p.endswith("benchmarks/aime24/config.yaml") for p in config_paths) + assert any(p.endswith("responses_api_models/openai_model/configs/openai_model.yaml") for p in config_paths) + assert others == {"++responses_create_params.reasoning.effort=low", "+wandb_project=gym-dev"} + + def test_dispatch_raises_on_unresolvable_attribute(self, monkeypatch: MonkeyPatch) -> None: + # A typo in a Command target must surface as a resolution error, not silently no-op. + monkeypatch.setattr(sys, "argv", ["gym"]) + with pytest.raises(AttributeError): + cli_main.dispatch("nemo_gym.cli.eval:no_such_function", []) + + class TestEvalProfileFlags: def test_profile_flags(self, monkeypatch: MonkeyPatch) -> None: target, overrides = _dispatch_for( From 14a790c333e6e148a4f99b3e871c151d91ed18cc Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Thu, 25 Jun 2026 10:16:09 +0200 Subject: [PATCH 39/40] chore: update docstrings to point to new commands Signed-off-by: Marta Stepniewska-Dziubinska --- benchmarks/finance_sec_search/prepare.py | 2 +- .../finance_sec_search/prepare_web_search.py | 2 +- benchmarks/gpqa/prepare.py | 2 +- benchmarks/graphwalks/prepare.py | 2 +- benchmarks/librispeech_pc/prepare.py | 4 +-- benchmarks/longbench_v2/prepare.py | 2 +- benchmarks/longcodebench/prepare.py | 2 +- benchmarks/m_arena_hard/prepare.py | 4 +-- benchmarks/mrcr/prepare.py | 2 +- benchmarks/polymath/tests/test_prepare.py | 2 +- benchmarks/ruler/ruler_prepare_script.py | 2 +- nemo_gym/cli/dataset.py | 2 +- nemo_gym/cli/dev.py | 2 +- nemo_gym/cli/env.py | 28 +++++++++---------- nemo_gym/cli/eval.py | 2 +- nemo_gym/cli/general.py | 4 +-- nemo_gym/cli/setup_command.py | 2 +- nemo_gym/config_types.py | 16 +++++------ nemo_gym/gitlab_utils.py | 2 +- nemo_gym/prompt.py | 2 +- nemo_gym/reward_profile.py | 4 +-- nemo_gym/rollout_collection.py | 14 +++++----- nemo_gym/train_data_utils.py | 2 +- resources_servers/bird_sql/eval_utils.py | 2 +- .../code_gen/livecodebench_accuracy_test.py | 2 +- .../livecodebench_accuracy_test_prep.py | 4 +-- .../scripts/preprocess_train_dataset.py | 4 +-- .../cvdp/scripts/cvdp_pass_at_k_report.py | 2 +- resources_servers/ifbench/setup_ifbench.py | 2 +- .../inverse_if/dataset_preprocess.py | 4 +-- .../multichallenge/dataset_preprocess.py | 2 +- resources_servers/newton_bench/client.py | 2 +- .../rdkit_chemistry/sandbox_launcher.py | 8 +++--- 33 files changed, 69 insertions(+), 69 deletions(-) diff --git a/benchmarks/finance_sec_search/prepare.py b/benchmarks/finance_sec_search/prepare.py index 2a6268238d..c040dbd8e6 100644 --- a/benchmarks/finance_sec_search/prepare.py +++ b/benchmarks/finance_sec_search/prepare.py @@ -228,7 +228,7 @@ def prepare(include_web_search: bool = False) -> Path: Args: include_web_search: Include the web_search tool in tool definitions. - Default False (ng_prepare_benchmark calls prepare() with no args). + Default False (gym eval prepare calls prepare() with no args). Pass True or use --include-web-search on the CLI. """ DATA_DIR.mkdir(parents=True, exist_ok=True) diff --git a/benchmarks/finance_sec_search/prepare_web_search.py b/benchmarks/finance_sec_search/prepare_web_search.py index 491a430cff..b0680cfea4 100644 --- a/benchmarks/finance_sec_search/prepare_web_search.py +++ b/benchmarks/finance_sec_search/prepare_web_search.py @@ -15,7 +15,7 @@ """Prepare finance_sec_search benchmark with web_search tool included. Thin wrapper around prepare.prepare() — used by config_web_search.yaml -so ng_prepare_benchmark produces the web_search variant. +so gym eval prepare produces the web_search variant. """ from pathlib import Path diff --git a/benchmarks/gpqa/prepare.py b/benchmarks/gpqa/prepare.py index 045732aff8..1dcf246dc9 100644 --- a/benchmarks/gpqa/prepare.py +++ b/benchmarks/gpqa/prepare.py @@ -19,7 +19,7 @@ compatible with the mcqa resource server. Output is raw data (no prompts baked in). Use prompt_config at rollout time -to specify the prompt, or ng_materialize_prompts to produce RL-ready data. +to specify the prompt, or gym dataset render to produce RL-ready data. """ import hashlib diff --git a/benchmarks/graphwalks/prepare.py b/benchmarks/graphwalks/prepare.py index cbbd963d8f..1aa5c88480 100644 --- a/benchmarks/graphwalks/prepare.py +++ b/benchmarks/graphwalks/prepare.py @@ -34,7 +34,7 @@ Invocation ---------- -``ng_prepare_benchmark`` calls ``prepare()`` with no arguments, using +``gym eval prepare`` calls ``prepare()`` with no arguments, using the defaults below. To build a custom variant, run this script directly:: diff --git a/benchmarks/librispeech_pc/prepare.py b/benchmarks/librispeech_pc/prepare.py index 63dba5b936..3aa9d64699 100644 --- a/benchmarks/librispeech_pc/prepare.py +++ b/benchmarks/librispeech_pc/prepare.py @@ -45,7 +45,7 @@ # test-clean is the standard LibriSpeech-PC eval split; test-other (~2.9k # harder utterances) can be enabled either by adding a sibling benchmark -# dir (the idiomatic shape, since `ng_prepare_benchmark` enforces one +# dir (the idiomatic shape, since `gym eval prepare` enforces one # dataset per agent) or by passing `--splits test-other` and pointing a # separate config at the resulting JSONL. Default below emits only # test-clean. @@ -173,7 +173,7 @@ def prepare(work_dir: Path | None = None, splits: tuple[str, ...] = DEFAULT_SPLI Returns: Path to ``librispeech_pc_test_clean.jsonl`` — the file the benchmark config's ``datasets[0].jsonl_fpath`` references. Returning a single - Path matches the contract ``ng_prepare_benchmark`` enforces. + Path matches the contract ``gym eval prepare`` enforces. """ work_dir = work_dir or DATA_DIR work_dir.mkdir(parents=True, exist_ok=True) diff --git a/benchmarks/longbench_v2/prepare.py b/benchmarks/longbench_v2/prepare.py index 702b73878f..e341fb344a 100644 --- a/benchmarks/longbench_v2/prepare.py +++ b/benchmarks/longbench_v2/prepare.py @@ -37,7 +37,7 @@ Invocation ---------- -``ng_prepare_benchmark`` calls ``prepare()`` with no arguments, using +``gym eval prepare`` calls ``prepare()`` with no arguments, using the defaults below. To build a custom variant, run this script directly:: diff --git a/benchmarks/longcodebench/prepare.py b/benchmarks/longcodebench/prepare.py index a650c2a539..dadf4b1bc8 100644 --- a/benchmarks/longcodebench/prepare.py +++ b/benchmarks/longcodebench/prepare.py @@ -35,7 +35,7 @@ Invocation ---------- -``ng_prepare_benchmark`` calls ``prepare()`` with no arguments, using +``gym eval prepare`` calls ``prepare()`` with no arguments, using the defaults below. To build a custom variant, run this script directly:: diff --git a/benchmarks/m_arena_hard/prepare.py b/benchmarks/m_arena_hard/prepare.py index bb92ff41b1..7b849b6f23 100644 --- a/benchmarks/m_arena_hard/prepare.py +++ b/benchmarks/m_arena_hard/prepare.py @@ -32,7 +32,7 @@ When ``--baseline-file`` is omitted, ``baseline_answer`` is set to the empty string so the JSONL is still well-formed for tooling that only needs the question set (e.g. data exploration or Skills-vs-Gym prepare -parity checks). ``ng_prepare_benchmark`` calls ``prepare()`` with no +parity checks). ``gym eval prepare`` calls ``prepare()`` with no arguments and so always takes this no-baseline path; use the script entry-point (``python prepare.py --baseline-file ...``) for full arena_judge runs. @@ -82,7 +82,7 @@ def _format_entry(row: dict, language: str) -> dict: def prepare(languages: list[str] | None = None, baseline_file: str | None = None) -> Path: """Download m-ArenaHard from HF, optionally join baselines, write JSONL. - Called with no arguments by ``ng_prepare_benchmark``. Returns the + Called with no arguments by ``gym eval prepare``. Returns the path to the written JSONL. """ DATA_DIR.mkdir(parents=True, exist_ok=True) diff --git a/benchmarks/mrcr/prepare.py b/benchmarks/mrcr/prepare.py index 2ef8e767b1..17b6d5d135 100644 --- a/benchmarks/mrcr/prepare.py +++ b/benchmarks/mrcr/prepare.py @@ -31,7 +31,7 @@ Invocation ---------- -``ng_prepare_benchmark`` calls ``prepare()`` with no arguments, using +``gym eval prepare`` calls ``prepare()`` with no arguments, using the defaults below. To build a custom variant, run this script directly:: diff --git a/benchmarks/polymath/tests/test_prepare.py b/benchmarks/polymath/tests/test_prepare.py index 1099bd5199..d57635fd46 100644 --- a/benchmarks/polymath/tests/test_prepare.py +++ b/benchmarks/polymath/tests/test_prepare.py @@ -16,7 +16,7 @@ Hits ``format_entry`` with hand-rolled instruction dicts; does NOT network out to HuggingFace or the upstream PolyMath repo (those are -covered by the on-cluster ``ng_prepare_benchmark`` step). +covered by the on-cluster ``gym eval prepare`` step). """ from benchmarks.polymath.prepare import DIFFICULTY_LEVEL_DIC, QUESTION_TEMPLATE, format_entry diff --git a/benchmarks/ruler/ruler_prepare_script.py b/benchmarks/ruler/ruler_prepare_script.py index 6e9d569f68..ab4deb29e0 100644 --- a/benchmarks/ruler/ruler_prepare_script.py +++ b/benchmarks/ruler/ruler_prepare_script.py @@ -152,7 +152,7 @@ def get_ruler_data( # `nltk.download(...)` branch entirely. # # The lock also coordinates across multiple `get_ruler_data` - # callers (e.g. `ng_prepare_benchmark` pool workers) on the same + # callers (e.g. `gym eval prepare` pool workers) on the same # host — `tempfile.gettempdir()` is `/tmp`, host-shared, and # `flock(2)` is a kernel-level lock, so it serializes correctly # across processes regardless of which Python venv they run in. diff --git a/nemo_gym/cli/dataset.py b/nemo_gym/cli/dataset.py index 0984b99993..bb2d211d50 100644 --- a/nemo_gym/cli/dataset.py +++ b/nemo_gym/cli/dataset.py @@ -75,7 +75,7 @@ def upload_jsonl_dataset_to_hf_and_delete_gitlab_cli() -> None: # pragma: no co def materialize_prompts_cli() -> None: # pragma: no cover - """CLI entry point for ng_materialize_prompts.""" + """CLI entry point for gym dataset render.""" global_config_dict = get_global_config_dict( global_config_dict_parser_config=GlobalConfigDictParserConfig( initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, diff --git a/nemo_gym/cli/dev.py b/nemo_gym/cli/dev.py index a960f11b5c..f0f5ca41fc 100644 --- a/nemo_gym/cli/dev.py +++ b/nemo_gym/cli/dev.py @@ -26,7 +26,7 @@ def dev_test(): # pragma: no cover Examples: ```bash - ng_dev_test + gym dev test ``` """ global_config_dict = get_global_config_dict() diff --git a/nemo_gym/cli/env.py b/nemo_gym/cli/env.py index b81bcb59f5..2e9f18a233 100644 --- a/nemo_gym/cli/env.py +++ b/nemo_gym/cli/env.py @@ -74,7 +74,7 @@ class RunConfig(BaseNeMoGymCLIConfig): ```bash config_paths="resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,\\ responses_api_models/openai_model/configs/openai_model.yaml" - ng_run "+config_paths=[${config_paths}]" + gym env start "+config_paths=[${config_paths}]" ``` """ @@ -90,7 +90,7 @@ class TestConfig(RunConfig): Examples: ```bash - ng_test +entrypoint=resources_servers/example_single_tool_call + gym env test +entrypoint=resources_servers/example_single_tool_call ``` """ @@ -410,7 +410,7 @@ def run( # Start servers with specific configs config_paths="resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,\\ responses_api_models/openai_model/configs/openai_model.yaml" - ng_run "+config_paths=[${config_paths}]" + gym env start "+config_paths=[${config_paths}]" ``` """ global_config_dict = get_global_config_dict(global_config_dict_parser_config=global_config_dict_parser_config) @@ -446,7 +446,7 @@ def _validate_data_single(test_config: TestConfig) -> None: # pragma: no cover ), f"""You must run the example data validation for the example data found at {example_fpath}. Your command should look something like the following (you should update this command with your actual server config path): ```bash -ng_prepare_data "+config_paths=[{test_config._dir_path}/configs/{server_type_name}.yaml]" \\ +gym dataset collate "+config_paths=[{test_config._dir_path}/configs/{server_type_name}.yaml]" \\ +output_dirpath=data/{server_type_name} \\ +mode=example_validation ``` @@ -482,10 +482,10 @@ def _validate_data_single(test_config: TestConfig) -> None: # pragma: no cover # Server spinup example_multi_step_config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ resources_servers/example_multi_step/configs/example_multi_step.yaml" -ng_run "+config_paths=[${{example_multi_step_config_paths}}]" +gym env start "+config_paths=[${{example_multi_step_config_paths}}]" # Collect the rollouts -ng_collect_rollouts +agent_name=example_multi_step_simple_agent \ +gym eval run --no-serve +agent_name=example_multi_step_simple_agent \ +input_jsonl_fpath=resources_servers/example_multi_step/data/example.jsonl \ +output_jsonl_fpath=resources_servers/example_multi_step/data/example_rollouts.jsonl \ +limit=null @@ -537,7 +537,7 @@ class TestAllConfig(BaseNeMoGymCLIConfig): Examples: ```bash - ng_test_all + gym env test ``` """ @@ -625,7 +625,7 @@ def test_all(): # pragma: no cover case _: raise ValueError( f"""Hit unrecognized exit code {return_code} while running tests for {dir_path}. -You can rerun just these tests using `ng_test +entrypoint={dir_path}` or run detailed tests via `cd {dir_path} && source .venv/bin/activate && pytest`.""" +You can rerun just these tests using `gym env test +entrypoint={dir_path}` or run detailed tests via `cd {dir_path} && source .venv/bin/activate && pytest`.""" ) try: @@ -664,13 +664,13 @@ def test_all(): # pragma: no cover if tests_failed or tests_missing: print(f"""You can rerun just the server with failed or missing tests like: ```bash -ng_test +entrypoint={(tests_failed + tests_missing)[0]} +gym env test +entrypoint={(tests_failed + tests_missing)[0]} ``` """) if data_validation_failed: print(f"""You can rerun just the server with failed data validation like: ```bash -ng_test +entrypoint={data_validation_failed[0]} +should_validate_data=true +gym env test +entrypoint={data_validation_failed[0]} +should_validate_data=true ``` """) @@ -695,7 +695,7 @@ def init_resources_server(): # pragma: no cover Examples: ```bash - ng_init_resources_server +entrypoint=resources_servers/my_server + gym env init +entrypoint=resources_servers/my_server ``` """ config_dict = get_global_config_dict() @@ -729,7 +729,7 @@ def init_resources_server(): # pragma: no cover verified: false # set true once the benchmark has been baselined and reviewed # Agent server config specifies the agent server to run and any additional components of the environment such as resources servers -{server_type_name}_simple_agent: # this agent instance's name — pass as +agent_name= to ng_collect_rollouts +{server_type_name}_simple_agent: # this agent instance's name — pass as --agent to gym eval run responses_api_agents: simple_agent: # built-in agent: runs the model with tool calls (up to max_steps); swap for your own agent dir entrypoint: app.py @@ -820,7 +820,7 @@ def dump_config(): # pragma: no cover Examples: ```bash - ng_dump_config "+config_paths=[,]" + gym env resolve "+config_paths=[,]" ``` """ global_config_dict = get_global_config_dict( @@ -871,7 +871,7 @@ def pip_list(): # pragma: no cover if not venv_path.exists(): print(f" Virtual environment not found at: {venv_path}") print(" Run tests or setup the server first using:") - print(f" ng_test +entrypoint={config.entrypoint}") + print(f" gym env test +entrypoint={config.entrypoint}") exit(1) pip_list_cmd = "uv pip list" diff --git a/nemo_gym/cli/eval.py b/nemo_gym/cli/eval.py index 1a8644ae50..8d3f5598df 100644 --- a/nemo_gym/cli/eval.py +++ b/nemo_gym/cli/eval.py @@ -191,7 +191,7 @@ class PrepareBenchmarkConfig(BaseNeMoGymCLIConfig): Examples: ```bash - ng_prepare_benchmark "+config_paths=[benchmarks/aime24/config.yaml]" + gym eval prepare --benchmark aime24 ``` """ diff --git a/nemo_gym/cli/general.py b/nemo_gym/cli/general.py index 29d7152ccf..615a5bab75 100644 --- a/nemo_gym/cli/general.py +++ b/nemo_gym/cli/general.py @@ -36,10 +36,10 @@ class VersionConfig(BaseNeMoGymCLIConfig): ```bash # Display version information - ng_version + gym --version # Output as JSON - ng_version +json=true + gym --version --json ``` """ diff --git a/nemo_gym/cli/setup_command.py b/nemo_gym/cli/setup_command.py index 3c979b07c5..c1940d4933 100644 --- a/nemo_gym/cli/setup_command.py +++ b/nemo_gym/cli/setup_command.py @@ -116,7 +116,7 @@ def setup_env_command(dir_path: Path, global_config_dict: DictConfig, prefix: st skip_venv_if_present = global_config_dict[SKIP_VENV_IF_PRESENT_KEY_NAME] should_skip_venv_setup = bool(skip_venv_if_present) and venv_python_fpath.exists() and venv_activate_fpath.exists() - # explicitly set python path if specified. In Google colab, ng_run fails due to uv pip install falls back to system python (/usr) without this and errors. + # explicitly set python path if specified. In Google colab, gym env start fails due to uv pip install falls back to system python (/usr) without this and errors. # not needed for most clusters. should be safe in all scenarios, but only minimally tested outside of colab. # see discussion and examples here: https://github.com/NVIDIA-NeMo/Gym/pull/526#issuecomment-3676230383 uv_pip_set_python = global_config_dict.get(UV_PIP_SET_PYTHON_KEY_NAME, False) diff --git a/nemo_gym/config_types.py b/nemo_gym/config_types.py index c80121ec43..75e9b59f54 100644 --- a/nemo_gym/config_types.py +++ b/nemo_gym/config_types.py @@ -156,7 +156,7 @@ class UploadJsonlDatasetGitlabConfig(BaseNeMoGymCLIConfig): Examples: ```bash - ng_upload_dataset_to_gitlab \ + gym dataset upload --storage gitlab \ +dataset_name=example_multi_step \ +version=0.0.1 \ +input_jsonl_fpath=data/train.jsonl @@ -181,7 +181,7 @@ class DownloadJsonlDatasetGitlabConfig(JsonlDatasetGitlabIdentifer, BaseNeMoGymC Examples: ```bash - ng_download_dataset_from_gitlab \ + gym dataset download --storage gitlab \ +dataset_name=example_multi_step \ +version=0.0.1 \ +artifact_fpath=train.jsonl \ @@ -202,7 +202,7 @@ class DeleteJsonlDatasetGitlabConfig(BaseNeMoGymCLIConfig): Examples: ```bash - ng_delete_dataset_from_gitlab +dataset_name=old_dataset + gym dataset rm +dataset_name=old_dataset ``` """ @@ -225,7 +225,7 @@ class BaseUploadJsonlDatasetHuggingFaceConfig(BaseNeMoGymCLIConfig): ```bash resource_config_path="resources_servers/example_multi_step/configs/example_multi_step.yaml" - ng_upload_dataset_to_hf \ + gym dataset upload \ +dataset_name=my_dataset \ +input_jsonl_fpath=data/train.jsonl \ +resource_config_path=${resource_config_path} @@ -272,13 +272,13 @@ class UploadJsonlDatasetHuggingFaceConfig(BaseUploadJsonlDatasetHuggingFaceConfi Upload a JSONL dataset to HuggingFace Hub and automatically delete from GitLab after successful upload. This command always deletes the dataset from GitLab after uploading to HuggingFace. - Use `ng_upload_dataset_to_hf` if you want optional deletion control. + Use `gym dataset upload` if you want optional deletion control. Examples: ```bash resource_config_path="resources_servers/example_multi_step/configs/example_multi_step.yaml" - ng_gitlab_to_hf_dataset \ + gym dataset migrate \ +dataset_name=my_dataset \ +input_jsonl_fpath=data/train.jsonl \ +resource_config_path=${resource_config_path} @@ -304,7 +304,7 @@ class UploadJsonlDatasetHuggingFaceMaybeDeleteConfig(BaseUploadJsonlDatasetHuggi ```bash resource_config_path="resources_servers/example_multi_step/configs/example_multi_step.yaml" - ng_upload_dataset_to_hf \ + gym dataset upload \ +dataset_name=my_dataset \ +input_jsonl_fpath=data/train.jsonl \ +resource_config_path=${resource_config_path} \ @@ -324,7 +324,7 @@ class DownloadJsonlDatasetHuggingFaceConfig(JsonlDatasetHuggingFaceIdentifer, Ba Examples: ```bash - ng_download_dataset_from_hf \ + gym dataset download \ +repo_id=NVIDIA/NeMo-Gym-Math-example_multi_step-v1 \ +artifact_fpath=train.jsonl \ +output_fpath=data/train.jsonl diff --git a/nemo_gym/gitlab_utils.py b/nemo_gym/gitlab_utils.py index b96093e0bc..4404a4e59c 100644 --- a/nemo_gym/gitlab_utils.py +++ b/nemo_gym/gitlab_utils.py @@ -66,7 +66,7 @@ def upload_jsonl_dataset( filename = Path(config.input_jsonl_fpath).name DownloadJsonlDatasetGitlabConfig print(f"""Download this artifact: -ng_download_dataset_from_gitlab \\ +gym dataset download --storage gitlab \\ +dataset_name={config.dataset_name} \\ +version={config.version} \\ +artifact_fpath={filename} \\ diff --git a/nemo_gym/prompt.py b/nemo_gym/prompt.py index 1fcb45990a..ed07d80eb0 100644 --- a/nemo_gym/prompt.py +++ b/nemo_gym/prompt.py @@ -151,7 +151,7 @@ class MaterializePromptsConfig(BaseNeMoGymCLIConfig): Examples: ```bash - ng_materialize_prompts \\ + gym dataset render \\ +input_jsonl_fpath=data/my_dataset.jsonl \\ +prompt_config=/path/to/my_prompt.yaml \\ +output_jsonl_fpath=my_dataset_materialized.jsonl diff --git a/nemo_gym/reward_profile.py b/nemo_gym/reward_profile.py index ba3f0e992e..1de3d354f0 100644 --- a/nemo_gym/reward_profile.py +++ b/nemo_gym/reward_profile.py @@ -36,9 +36,9 @@ class RewardProfileConfig(BaseNeMoGymCLIConfig): materialized_inputs_jsonl_fpath: str = Field( - description="The file path of the materialized inputs as output by ng_collect_rollouts." + description="The file path of the materialized inputs as output by `gym eval run`." ) - rollouts_jsonl_fpath: str = Field(description="The file path of the rollouts as output by ng_collect_rollouts.") + rollouts_jsonl_fpath: str = Field(description="The file path of the rollouts as output by `gym eval run`.") allow_partial_rollouts: bool = Field( default=False, description="Allow reward profiling from partial rollout outputs by dropping input rows with no completed rollouts.", diff --git a/nemo_gym/rollout_collection.py b/nemo_gym/rollout_collection.py index 72a0ac9378..1aa7d3c08c 100644 --- a/nemo_gym/rollout_collection.py +++ b/nemo_gym/rollout_collection.py @@ -127,7 +127,7 @@ class SharedRolloutCollectionConfig(BaseNeMoGymCLIConfig): description=( "Skip the post-rollout aggregate-metrics computation and file write. " "Used when sharding rollouts across multiple jobs that will be aggregated together " - "afterward by `ng_aggregate_rollouts`." + "afterward by `gym eval aggregate`." ), ) @@ -139,7 +139,7 @@ class E2ERolloutCollectionConfig(SharedRolloutCollectionConfig): Examples: ```bash - ng_collect_rollouts \ + gym eval run \ +output_jsonl_fpath=weather_rollouts.jsonl \ +num_samples_in_parallel=10 ``` @@ -156,7 +156,7 @@ class RolloutCollectionConfig(SharedRolloutCollectionConfig): Examples: ```bash - ng_collect_rollouts \ + gym eval run --no-serve \ +agent_name=example_single_tool_call_simple_agent \ +input_jsonl_fpath=weather_query.jsonl \ +output_jsonl_fpath=weather_rollouts.jsonl \ @@ -256,7 +256,7 @@ def _preprocess_rows_from_config(self, config: RolloutCollectionConfig) -> List[ validate_prompt_compatibility([row for _, _, row in raw_rows], prompt_cfg) raw_rows = [(idx, s, apply_prompt_to_row(row, prompt_cfg)) for idx, s, row in raw_rows] - # For ng_reward_profile to match rollouts to tasks + # For gym eval profile to match rollouts to tasks row_to_task_idx: Dict[str, int] = dict() task_idx_to_rollout_idx: Dict[int, int] = Counter() row_idxs_missing_agent_ref: List[int] = [] @@ -463,7 +463,7 @@ async def run_from_config(self, config: RolloutCollectionConfig) -> Tuple[List[D if config.disable_aggregation: print( "Skipping aggregate-metrics computation because disable_aggregation=True. " - "Run `ng_aggregate_rollouts` after all shards finish to compute the global metrics." + "Run `gym eval aggregate` after all shards finish to compute the global metrics." ) aggregate_metrics_fpath = None else: @@ -618,7 +618,7 @@ def setup_server_client( class RolloutAggregationConfig(BaseNeMoGymCLIConfig): """ - Aggregate metrics across rollout shards produced by `ng_collect_rollouts +disable_aggregation=true`. + Aggregate metrics across rollout shards produced by `gym eval run --no-serve +disable_aggregation=true`. Reads every JSONL file matching `input_glob`, computes aggregate metrics by POSTing to each agent server's `/aggregate_metrics` endpoint over the global union of records, and writes a @@ -628,7 +628,7 @@ class RolloutAggregationConfig(BaseNeMoGymCLIConfig): Examples: ```bash - ng_aggregate_rollouts \ + gym eval aggregate \ "+config_paths=[benchmarks/aime24/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ +input_glob='results/rollouts-rs*-chunk*.jsonl' \ +output_jsonl_fpath=results/rollouts.jsonl diff --git a/nemo_gym/train_data_utils.py b/nemo_gym/train_data_utils.py index 56c33fecd9..c632757e53 100644 --- a/nemo_gym/train_data_utils.py +++ b/nemo_gym/train_data_utils.py @@ -57,7 +57,7 @@ class TrainDataProcessorConfig(BaseNeMoGymCLIConfig): ```bash config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\\ responses_api_models/openai_model/configs/openai_model.yaml" - ng_prepare_data "+config_paths=[${config_paths}]" \ + gym dataset collate "+config_paths=[${config_paths}]" \ +output_dirpath=data/example_multi_step \ +mode=example_validation ``` diff --git a/resources_servers/bird_sql/eval_utils.py b/resources_servers/bird_sql/eval_utils.py index 7125379800..3acac2eb7f 100644 --- a/resources_servers/bird_sql/eval_utils.py +++ b/resources_servers/bird_sql/eval_utils.py @@ -8,7 +8,7 @@ SQL execution uses ``asyncio.to_thread`` rather than Ray remote tasks. When this resource server runs alongside a Ray-coordinated multi-node DP -vLLM (Gym's ``ng_run`` attaches to the same Ray cluster), the vLLM engines +vLLM (``gym env start`` attaches to the same Ray cluster), the vLLM engines consume the Ray slots and ``@ray.remote`` sqlite tasks sit in the scheduler queue past the per-query timeout, get cancelled, and every rollout reports ``gold_execution_error``. SQLite queries are fast and self-contained — no diff --git a/resources_servers/code_gen/livecodebench_accuracy_test.py b/resources_servers/code_gen/livecodebench_accuracy_test.py index 25562a0acb..d8e4011f66 100644 --- a/resources_servers/code_gen/livecodebench_accuracy_test.py +++ b/resources_servers/code_gen/livecodebench_accuracy_test.py @@ -26,7 +26,7 @@ ```bash config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ resources_servers/code_gen/configs/code_gen.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start "+config_paths=[${config_paths}]" ``` """ diff --git a/resources_servers/code_gen/livecodebench_accuracy_test_prep.py b/resources_servers/code_gen/livecodebench_accuracy_test_prep.py index 49db32c4a9..beb2c68d4a 100644 --- a/resources_servers/code_gen/livecodebench_accuracy_test_prep.py +++ b/resources_servers/code_gen/livecodebench_accuracy_test_prep.py @@ -57,13 +57,13 @@ Download the test data using: ```bash -ng_download_dataset_from_gitlab \ +gym dataset download --storage gitlab \ +dataset_name=livecodebench \ +version=0.0.1 \ +artifact_fpath=gpt-4o-2024-05-13_Scenario.codegeneration_10_0.2_eval_all.json \ +output_fpath=resources_servers/code_gen/data/gpt-4o-2024-05-13_Scenario.codegeneration_10_0.2_eval_all.json -ng_download_dataset_from_gitlab \ +gym dataset download --storage gitlab \ +dataset_name=livecodebench \ +version=0.0.1 \ +artifact_fpath=livecodebench_prompts.jsonl \ diff --git a/resources_servers/code_gen/scripts/preprocess_train_dataset.py b/resources_servers/code_gen/scripts/preprocess_train_dataset.py index 429079910a..3c50bd16f6 100644 --- a/resources_servers/code_gen/scripts/preprocess_train_dataset.py +++ b/resources_servers/code_gen/scripts/preprocess_train_dataset.py @@ -23,7 +23,7 @@ Upload: ```bash -ng_upload_dataset_to_gitlab \ +gym dataset upload --storage gitlab \ +dataset_name=opencodereasoning_filtered \ +version=0.0.1 \ +input_jsonl_fpath=resources_servers/code_gen/data/opencodereasoning_filtered_25k_train.jsonl @@ -31,7 +31,7 @@ Rollout collection. We match the LCB setting for reward profiling. For gpt-4o-2024-05-13 this should be around 33%. ```bash -ng_collect_rollouts +agent_name=code_gen_simple_agent \ +gym eval run --no-serve +agent_name=code_gen_simple_agent \ +input_jsonl_fpath=resources_servers/code_gen/data/opencodereasoning_filtered_25k_train.jsonl \ +output_jsonl_fpath=resources_servers/code_gen/data/opencodereasoning_filtered_25k_train_1k_gpt-4o-2024-05-13_rollouts.jsonl \ +responses_create_params.temperature=0.2 \ diff --git a/resources_servers/cvdp/scripts/cvdp_pass_at_k_report.py b/resources_servers/cvdp/scripts/cvdp_pass_at_k_report.py index 6d50b33c8a..2c25cb9b4c 100644 --- a/resources_servers/cvdp/scripts/cvdp_pass_at_k_report.py +++ b/resources_servers/cvdp/scripts/cvdp_pass_at_k_report.py @@ -47,7 +47,7 @@ def split_by_rollout(rollouts_path: str) -> dict[int, list[dict]]: def main() -> None: parser = argparse.ArgumentParser(description="Generate CVDP pass@k report from NeMo-Gym rollouts") - parser.add_argument("--rollouts", required=True, help="pass@k rollout JSONL from ng_collect_rollouts") + parser.add_argument("--rollouts", required=True, help="pass@k rollout JSONL from gym eval run") parser.add_argument("--output", required=True, help="Output directory") parser.add_argument("--model", default="nemo-gym", help="Model name for report metadata") parser.add_argument("--dataset", help="Path to original CVDP dataset JSONL (for report metadata)") diff --git a/resources_servers/ifbench/setup_ifbench.py b/resources_servers/ifbench/setup_ifbench.py index b610d8e11d..5e9948b372 100644 --- a/resources_servers/ifbench/setup_ifbench.py +++ b/resources_servers/ifbench/setup_ifbench.py @@ -20,7 +20,7 @@ This way, we can just `import instructions_registry`. IFbench dependencies (spaCy, nltk, syllapy, etc.) are listed in requirements.txt -and installed during `ng_run`. The `.installed` marker tracks whether the git +and installed during `gym env start`. The `.installed` marker tracks whether the git clone has been done. """ diff --git a/resources_servers/inverse_if/dataset_preprocess.py b/resources_servers/inverse_if/dataset_preprocess.py index bd510ef956..1c56ff65c7 100644 --- a/resources_servers/inverse_if/dataset_preprocess.py +++ b/resources_servers/inverse_if/dataset_preprocess.py @@ -27,7 +27,7 @@ - Normalises inconsistent rubric key names (criteria, criteria1, rule, question, …) into a uniform ``criteria`` field - Falls back to parsing ``response_reference`` when the ``rubrics`` array is empty -- Wraps the prompt in ``responses_create_params`` for ``ng_collect_rollouts`` +- Wraps the prompt in ``responses_create_params`` for ``gym eval run`` """ import argparse @@ -193,7 +193,7 @@ def process_task(task: dict, fallback_id: str = "unknown") -> dict[str, Any]: "type": "responses_api_agents", "name": "inverse_if_simple_agent", }, - # Input wrapped for ng_collect_rollouts + # Input wrapped for gym eval run "responses_create_params": { "input": [{"role": "user", "content": prompt}], }, diff --git a/resources_servers/multichallenge/dataset_preprocess.py b/resources_servers/multichallenge/dataset_preprocess.py index 798848e16b..cdf945f033 100644 --- a/resources_servers/multichallenge/dataset_preprocess.py +++ b/resources_servers/multichallenge/dataset_preprocess.py @@ -101,7 +101,7 @@ def process_task(task: dict, fallback_id: str = "unknown") -> dict[str, Any]: "type": "responses_api_agents", "name": "multichallenge_simple_agent", }, - # Input messages wrapped in responses_create_params (required by ng_collect_rollouts) + # Input messages wrapped in responses_create_params (required by gym eval run) "responses_create_params": { "input": build_input_messages(task), }, diff --git a/resources_servers/newton_bench/client.py b/resources_servers/newton_bench/client.py index 5020af3fef..43628cce24 100644 --- a/resources_servers/newton_bench/client.py +++ b/resources_servers/newton_bench/client.py @@ -24,7 +24,7 @@ 4. Discover the underlying physical law Prerequisites: -- NewtonBench server running via ng_run +- NewtonBench server running via gym env start - NewtonBench repo cloned in project root (for run_experiment to work) Usage: diff --git a/resources_servers/rdkit_chemistry/sandbox_launcher.py b/resources_servers/rdkit_chemistry/sandbox_launcher.py index 36bc14fedf..ba469196df 100644 --- a/resources_servers/rdkit_chemistry/sandbox_launcher.py +++ b/resources_servers/rdkit_chemistry/sandbox_launcher.py @@ -211,7 +211,7 @@ def start_sandbox( python = os.path.join(venv_path, "bin", "python") pip = os.path.join(venv_path, "bin", "pip") - # ng_run creates all server venvs in parallel. The ns_tools venv + # gym env start creates all server venvs in parallel. The ns_tools venv # may not be ready yet when rdkit_chemistry starts — wait for it. _wait_for_venv(python) @@ -298,13 +298,13 @@ def _run_startup_probe(port: int, timeout_s: float = _DEFAULT_STARTUP_PROBE_TIME logger.info("Sandbox startup probe passed on 127.0.0.1:%d", port) -_VENV_TIMEOUT = 600.0 # ng_run venv creation can take several minutes +_VENV_TIMEOUT = 600.0 # gym env start venv creation can take several minutes def _wait_for_venv(python: str) -> None: """Block until the venv's python binary exists and nemo_skills is importable. - ng_run creates all server venvs concurrently, so the ns_tools venv (which + gym env start creates all server venvs concurrently, so the ns_tools venv (which has nemo_skills) may still be installing when rdkit_chemistry starts. """ deadline = time.monotonic() + _VENV_TIMEOUT @@ -319,7 +319,7 @@ def _wait_for_venv(python: str) -> None: else: raise FileNotFoundError( f"Sandbox venv python not found at {python} after {_VENV_TIMEOUT}s. " - "Ensure ns_tools is part of the ng_run config." + "Ensure ns_tools is part of the gym env start config." ) phase = "nemo_skills" From 2775550d740c0f4c34504d8e06d6f34ddeb9b66e Mon Sep 17 00:00:00 2001 From: Marta Stepniewska-Dziubinska Date: Thu, 25 Jun 2026 11:45:44 +0200 Subject: [PATCH 40/40] feat: test if legacy commands point to real new commands Signed-off-by: Marta Stepniewska-Dziubinska --- tests/unit_tests/test_cli_legacy.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/unit_tests/test_cli_legacy.py b/tests/unit_tests/test_cli_legacy.py index 1589b09a85..4fe01e6f9d 100644 --- a/tests/unit_tests/test_cli_legacy.py +++ b/tests/unit_tests/test_cli_legacy.py @@ -20,6 +20,7 @@ from pytest import MonkeyPatch import nemo_gym.cli.legacy as legacy +import nemo_gym.cli.main as cli_main from nemo_gym import PARENT_DIR @@ -54,6 +55,26 @@ def test_legacy_command_shows_deprecation(self, monkeypatch: MonkeyPatch, capsys assert "deprecated" in capsys.readouterr().err + def test_legacy_mapping_is_populated(self) -> None: + # Guard so the parametrized test below can't pass vacuously if LEGACY is emptied. + assert len(legacy.LEGACY) > 1 + + @pytest.mark.parametrize("key, tokens", list(legacy.LEGACY.items())) + def test_legacy_tokens_resolve_to_real_gym_command( + self, monkeypatch: MonkeyPatch, key: str, tokens: list[str] + ) -> None: + # The tests above confirm each alias routes through the shim, but not that the mapped tokens + # are a real `gym` command. Feed every mapping through the actual parser so a typo in the + # mapping (e.g. ["env", "ruun"]) fails here instead of only at user runtime. + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", *tokens]) + try: + cli_main.main() + except SystemExit as exc: + # `gym --help` is the only mapping that legitimately exits (argparse prints help, exit 0). + assert tokens == ["--help"], f"`gym {' '.join(tokens)}` (legacy `{key}`) is not a valid command" + assert exc.code == 0 + def test_unknown_alias_exits_nonzero(self, monkeypatch: MonkeyPatch, capsys) -> None: # An alias with no LEGACY mapping (stale script or user typo) must fail loudly, not KeyError. monkeypatch.setattr(legacy, "gym_main", lambda: None)