diff --git a/.claude/skills/add-benchmark/SKILL.md b/.claude/skills/add-benchmark/SKILL.md index 385666e384..eaa4334bdb 100644 --- a/.claude/skills/add-benchmark/SKILL.md +++ b/.claude/skills/add-benchmark/SKILL.md @@ -32,10 +32,10 @@ Before starting, determine which type of benchmark you're adding: ### Step 1: Scaffold the server -Run `ng_init_resources_server` to generate the directory structure: +Run `gym env init` to generate the directory structure: ```bash -ng_init_resources_server +entrypoint=resources_servers/my_benchmark +gym env init --resources-server my_benchmark ``` This creates: @@ -77,10 +77,10 @@ Convert your source dataset to Gym JSONL format. Each line must have `responses_ **`train`/`validation` datasets**: Upload to the GitLab dataset registry — these must NOT be committed to git. ```bash -ng_upload_dataset_to_gitlab \ - +dataset_name=my_benchmark \ - +version=0.0.1 \ - +input_jsonl_fpath=resources_servers/my_benchmark/data/my_dataset.jsonl +gym dataset upload --storage gitlab \ + --name my_benchmark \ + --revision 0.0.1 \ + --input resources_servers/my_benchmark/data/my_dataset.jsonl ``` Requires MLflow credentials in `env.yaml` (or passed via CLI): @@ -94,12 +94,16 @@ mlflow_tracking_token: **Validate** your data: ```bash # Validate example data (for PR submission) -ng_prepare_data "+config_paths=[resources_servers/my_benchmark/configs/my_benchmark.yaml]" \ - +output_dirpath=/tmp/prepare +mode=example_validation +gym dataset collate --config resources_servers/my_benchmark/configs/my_benchmark.yaml \ + --output-dir /tmp/prepare \ + --mode example_validation # Download and prepare train/validation from GitLab -ng_prepare_data "+config_paths=[resources_servers/my_benchmark/configs/my_benchmark.yaml]" \ - +output_dirpath=data/my_benchmark +mode=train_preparation +should_download=true +data_source=gitlab +gym dataset collate --config resources_servers/my_benchmark/configs/my_benchmark.yaml \ + --output-dir data/my_benchmark \ + --mode train_preparation \ + --download \ + +data_source=gitlab ``` ### Step 3: Implement verify() @@ -162,7 +166,7 @@ Both fields must coexist: `jsonl_fpath` is the local download destination, `gitl ```bash # Run server tests (creates isolated .venv, slow on first run) -ng_test +entrypoint=resources_servers/my_benchmark +gym env test --resources-server my_benchmark # Run core library tests to check nothing broke pytest tests/unit_tests/ -x @@ -174,14 +178,18 @@ Test coverage must be >= 95%. Write tests for: verify pass, verify fail (wrong o ```bash # Start servers -ng_run "+config_paths=[resources_servers/my_benchmark/configs/my_benchmark.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env start \ + --config resources_servers/my_benchmark/configs/my_benchmark.yaml \ + --model-type openai_model # Quick test with example data -ng_collect_rollouts +agent_name=my_benchmark_simple_agent \ - +input_jsonl_fpath=resources_servers/my_benchmark/data/example.jsonl \ - +output_jsonl_fpath=results/example_rollouts.jsonl \ - +num_repeats=1 \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent my_benchmark_simple_agent \ + --input resources_servers/my_benchmark/data/example.jsonl \ + --output results/example_rollouts.jsonl \ + --num-repeats 1 \ + --max-output-tokens 16384 \ + --temperature 1.0 # Inspect results ``` @@ -196,17 +204,17 @@ Run against multiple models to validate correctness. Recommended suite: ```bash # Collect rollouts -ng_collect_rollouts +agent_name=my_benchmark_simple_agent \ - +input_jsonl_fpath=resources_servers/my_benchmark/data/my_dataset.jsonl \ - +output_jsonl_fpath=results/rollouts.jsonl \ - +num_repeats=5 \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent my_benchmark_simple_agent \ + --input resources_servers/my_benchmark/data/my_dataset.jsonl \ + --output results/rollouts.jsonl \ + --num-repeats 5 \ + --max-output-tokens 16384 \ + --temperature 1.0 # Compute per-task pass rates -ng_reward_profile +input_jsonl_fpath=resources_servers/my_benchmark/data/my_dataset.jsonl \ - +rollouts_jsonl_fpath=results/rollouts.jsonl \ - +output_jsonl_fpath=results/profiled.jsonl \ - +pass_threshold=1.0 +gym eval profile --inputs resources_servers/my_benchmark/data/my_dataset.jsonl \ + --rollouts results/rollouts.jsonl # Aggregate metrics (pass@1 = avg_reward, pass@k from max_reward) python scripts/print_aggregate_results.py +jsonl_fpath=results/profiled.jsonl diff --git a/.claude/skills/add-benchmark/references/patterns.md b/.claude/skills/add-benchmark/references/patterns.md index 68d3668b9d..4c3a2c0fc9 100644 --- a/.claude/skills/add-benchmark/references/patterns.md +++ b/.claude/skills/add-benchmark/references/patterns.md @@ -546,7 +546,7 @@ Externalize system prompts to text files, pass via `--system-prompt` argument. M ### `data/.gitignore` default patterns -Generated by `ng_init_resources_server`: +Generated by `gym env init`: ``` *train.jsonl *validation.jsonl @@ -562,10 +562,10 @@ If your filename doesn't match (e.g. `my_eval.jsonl`), add a custom pattern (e.g 1. **Generate** the JSONL file using your conversion script (in the source repo) 2. **Upload** to GitLab dataset registry: ```bash - ng_upload_dataset_to_gitlab \ - +dataset_name=my_benchmark \ - +version=0.0.1 \ - +input_jsonl_fpath=resources_servers/my_benchmark/data/my_dataset.jsonl + gym dataset upload --storage gitlab \ + --name my_benchmark \ + --revision 0.0.1 \ + --input resources_servers/my_benchmark/data/my_dataset.jsonl ``` 3. **Add `gitlab_identifier`** to the dataset entry in YAML config: ```yaml @@ -591,18 +591,22 @@ mlflow_tracking_token: The tracking URI format is `https:///api/v4/projects//ml/mlflow`. -### Verification with `ng_prepare_data` +### Verification with `gym dataset collate` Validate example data (for PR submission): ```bash -ng_prepare_data "+config_paths=[resources_servers/my_benchmark/configs/my_benchmark.yaml]" \ - +output_dirpath=/tmp/prepare +mode=example_validation +gym dataset collate --config resources_servers/my_benchmark/configs/my_benchmark.yaml \ + --output-dir /tmp/prepare \ + --mode example_validation ``` Download and prepare train/validation from GitLab: ```bash -ng_prepare_data "+config_paths=[resources_servers/my_benchmark/configs/my_benchmark.yaml]" \ - +output_dirpath=data/my_benchmark +mode=train_preparation +should_download=true +data_source=gitlab +gym dataset collate --config resources_servers/my_benchmark/configs/my_benchmark.yaml \ + --output-dir data/my_benchmark \ + --mode train_preparation \ + --download \ + +data_source=gitlab ``` --- diff --git a/.claude/skills/nemo-gym-debugging/references/error-profiles.md b/.claude/skills/nemo-gym-debugging/references/error-profiles.md index 666b9eb9d4..9705582aec 100644 --- a/.claude/skills/nemo-gym-debugging/references/error-profiles.md +++ b/.claude/skills/nemo-gym-debugging/references/error-profiles.md @@ -113,7 +113,7 @@ Next actions: Symptoms: -- `ng_run` or `ng_collect_rollouts` fails before launching servers +- `gym env start` or `gym eval run --no-serve` fails before launching servers - errors mention missing config keys, unknown overrides, or interpolation failures - wrong agent/resource server starts despite expected data @@ -134,7 +134,7 @@ Evidence to collect: Next actions: -- reduce to a minimal `ng_run "+config_paths=[...]"` command +- reduce to a minimal `gym env start --config ...` command - add overrides back one at a time - trust target checkout code over copied command templates diff --git a/.claude/skills/nemo-gym-debugging/references/request-boundary-visibility.md b/.claude/skills/nemo-gym-debugging/references/request-boundary-visibility.md index c992750f40..a56b2f1ee9 100644 --- a/.claude/skills/nemo-gym-debugging/references/request-boundary-visibility.md +++ b/.claude/skills/nemo-gym-debugging/references/request-boundary-visibility.md @@ -9,7 +9,7 @@ This is an escalation ladder. Do not patch agents, resource servers, model adapt 1. Enable Gym's existing request debug flag: ```bash - ng_run '+config_paths=[...]' ++global_aiohttp_client_request_debug=True + gym env start --config ... ++global_aiohttp_client_request_debug=True ``` 2. Re-run the smallest failing case and capture stdout/stderr. diff --git a/.claude/skills/nemo-gym-reward-profiling/SKILL.md b/.claude/skills/nemo-gym-reward-profiling/SKILL.md index 1d177d087f..ebcb7758db 100644 --- a/.claude/skills/nemo-gym-reward-profiling/SKILL.md +++ b/.claude/skills/nemo-gym-reward-profiling/SKILL.md @@ -2,7 +2,7 @@ name: nemo-gym-reward-profiling description: >- Use to help users get started with Nemo Gym reward profiling. Covers the basic - ng_run, ng_collect_rollouts, and ng_reward_profile workflow, repeated rollouts, + gym env start, gym eval run, and gym eval profile workflow, repeated rollouts, materialized inputs, rollout JSONL artifacts, task and rollout identity, output inspection, partial profiling, and rollout_infos. For failed jobs, prefer nemo-gym-debugging. @@ -14,16 +14,16 @@ description: >- Use this skill when the user wants to run, understand, or lightly modify Nemo Gym reward profiling. Keep the answer oriented around the normal workflow: -`ng_run` starts model/resource servers, `ng_collect_rollouts` writes rollout artifacts, and `ng_reward_profile` generates profiling output from those artifacts. +`gym env start` starts model/resources servers, `gym eval run --no-serve` writes rollout artifacts, and `gym eval profile` generates profiling output from those artifacts. If the user is primarily debugging a failed job or stack trace, use the `nemo-gym-debugging` skill first. ## Basic Workflow 1. Identify the environment config paths and input JSONL. -2. Start Gym servers with `ng_run`. -3. Collect rollouts with `ng_collect_rollouts`; this writes `rollouts.jsonl` and `*_materialized_inputs.jsonl`. -4. Run `ng_reward_profile` on the materialized inputs and rollout JSONL to generate `*_reward_profiling.jsonl`. +2. Start Gym servers with `gym env start`. +3. Collect rollouts with `gym eval run --no-serve`; this writes `rollouts.jsonl` and `*_materialized_inputs.jsonl`. +4. Run `gym eval profile` on the materialized inputs and rollout JSONL to generate `*_reward_profiling.jsonl`. 5. Inspect line counts and profile rows. Repeated rollouts are the main profiling lever. `num_repeats=1` is valid, but per-task averages and variance are only meaningful with multiple rollouts per task. @@ -48,6 +48,6 @@ Load references only when the user needs that detail: ## Practical Defaults -- Treat `ng_reward_profile` as the reward profiling step; rollout collection does not write reward profile files. +- Treat `gym eval profile` as the reward profiling step; rollout collection does not write reward profile files. - Run strict profiling by default. If rollout collection stopped early, use `++allow_partial_rollouts=True` to profile completed rollouts and drop original input rows with no completed rollout. - Trust the target checkout's CLI help and `nemo_gym/reward_profile.py` over memory if flags differ. diff --git a/.claude/skills/nemo-gym-reward-profiling/references/output-format.md b/.claude/skills/nemo-gym-reward-profiling/references/output-format.md index 71ab7b572d..23ab8dbfd3 100644 --- a/.claude/skills/nemo-gym-reward-profiling/references/output-format.md +++ b/.claude/skills/nemo-gym-reward-profiling/references/output-format.md @@ -6,7 +6,7 @@ Reward profiling is built around joining materialized inputs with rollout result - `*_materialized_inputs.jsonl`: expanded inputs after repeat expansion. Each row should have `_ng_task_index` and `_ng_rollout_index`. - `rollouts.jsonl`: completed rollout results. Each row should have matching `_ng_task_index` and `_ng_rollout_index`. -- `*_reward_profiling.jsonl`: task-level summaries produced by `ng_reward_profile`. +- `*_reward_profiling.jsonl`: task-level summaries produced by `gym eval profile`. - `*_agent_metrics.json`: agent/global aggregate metrics. ## Task Profile Rows @@ -36,7 +36,7 @@ Full model responses stay in `rollouts.jsonl`. Join back to full rows with `(_ng ## Partial Profiles -Strict profiling is the default. If materialized inputs and rollout results do not have the same rollout keys, `ng_reward_profile` fails and suggests: +Strict profiling is the default. If materialized inputs and rollout results do not have the same rollout keys, `gym eval profile` fails and suggests: ```bash ++allow_partial_rollouts=True diff --git a/.claude/skills/nemo-gym-reward-profiling/references/quick-start.md b/.claude/skills/nemo-gym-reward-profiling/references/quick-start.md index 0e2c80838b..3b19d2e832 100644 --- a/.claude/skills/nemo-gym-reward-profiling/references/quick-start.md +++ b/.claude/skills/nemo-gym-reward-profiling/references/quick-start.md @@ -5,8 +5,6 @@ Substitute environment-specific config paths, input data, model endpoint, and ou ## Minimal Flow ```bash -CONFIG_PATHS="your_model_config_paths,your_env_config_paths" - POLICY_MODEL_NAME="your_policy_model_name" POLICY_BASE_URL="your_policy_base_url" POLICY_ENDPOINT_KEY="your_policy_endpoint_key" @@ -19,41 +17,43 @@ AGENT_NAME="your_agent_name" NUM_REPEATS=2 NUM_SAMPLES_IN_PARALLEL=8 -ng_run "+config_paths=[$CONFIG_PATHS]" \ - +policy_model_name="$POLICY_MODEL_NAME" \ - +policy_base_url="$POLICY_BASE_URL" \ - +policy_api_key="$POLICY_ENDPOINT_KEY" & +gym env start \ + --config your_model_config_paths \ + --config your_env_config_paths \ + --model "$POLICY_MODEL_NAME" \ + --model-url "$POLICY_BASE_URL" \ + --model-api-key "$POLICY_ENDPOINT_KEY" & NG_RUN_PID=$! trap 'kill "$NG_RUN_PID" 2>/dev/null || true' EXIT ./scripts/wait_for_servers.sh "$NG_RUN_PID" agent_args=() if [[ -n "$AGENT_NAME" ]]; then - agent_args=(+agent_name="$AGENT_NAME") + agent_args=(--agent "$AGENT_NAME") fi -ng_collect_rollouts \ +gym eval run --no-serve \ "${agent_args[@]}" \ - +input_jsonl_fpath="$DATA_JSONL" \ - +output_jsonl_fpath="$ROLLOUTS_JSONL" \ - +num_repeats="$NUM_REPEATS" \ - +num_samples_in_parallel="$NUM_SAMPLES_IN_PARALLEL" - -ng_reward_profile \ - ++materialized_inputs_jsonl_fpath="$MATERIALIZED_JSONL" \ - ++rollouts_jsonl_fpath="$ROLLOUTS_JSONL" + --input "$DATA_JSONL" \ + --output "$ROLLOUTS_JSONL" \ + --num-repeats "$NUM_REPEATS" \ + --concurrency "$NUM_SAMPLES_IN_PARALLEL" + +gym eval profile \ + --inputs "$MATERIALIZED_JSONL" \ + --rollouts "$ROLLOUTS_JSONL" ``` If rows already contain `agent_ref`, leave `AGENT_NAME` empty. Passing `+agent_name` supplies a default for rows without one. ## Partial Rollouts -By default, `ng_reward_profile` expects every materialized input row to have a matching rollout row. If collection stopped early, profile the completed rollouts with: +By default, `gym eval profile` expects every materialized input row to have a matching rollout row. If collection stopped early, profile the completed rollouts with: ```bash -ng_reward_profile \ - ++materialized_inputs_jsonl_fpath="$MATERIALIZED_JSONL" \ - ++rollouts_jsonl_fpath="$ROLLOUTS_JSONL" \ +gym eval profile \ + --inputs "$MATERIALIZED_JSONL" \ + --rollouts "$ROLLOUTS_JSONL" \ ++allow_partial_rollouts=True ``` diff --git a/.codex/skills/nemo-gym-debugging/references/error-profiles.md b/.codex/skills/nemo-gym-debugging/references/error-profiles.md index 666b9eb9d4..9705582aec 100644 --- a/.codex/skills/nemo-gym-debugging/references/error-profiles.md +++ b/.codex/skills/nemo-gym-debugging/references/error-profiles.md @@ -113,7 +113,7 @@ Next actions: Symptoms: -- `ng_run` or `ng_collect_rollouts` fails before launching servers +- `gym env start` or `gym eval run --no-serve` fails before launching servers - errors mention missing config keys, unknown overrides, or interpolation failures - wrong agent/resource server starts despite expected data @@ -134,7 +134,7 @@ Evidence to collect: Next actions: -- reduce to a minimal `ng_run "+config_paths=[...]"` command +- reduce to a minimal `gym env start --config ...` command - add overrides back one at a time - trust target checkout code over copied command templates diff --git a/.codex/skills/nemo-gym-debugging/references/request-boundary-visibility.md b/.codex/skills/nemo-gym-debugging/references/request-boundary-visibility.md index c992750f40..a56b2f1ee9 100644 --- a/.codex/skills/nemo-gym-debugging/references/request-boundary-visibility.md +++ b/.codex/skills/nemo-gym-debugging/references/request-boundary-visibility.md @@ -9,7 +9,7 @@ This is an escalation ladder. Do not patch agents, resource servers, model adapt 1. Enable Gym's existing request debug flag: ```bash - ng_run '+config_paths=[...]' ++global_aiohttp_client_request_debug=True + gym env start --config ... ++global_aiohttp_client_request_debug=True ``` 2. Re-run the smallest failing case and capture stdout/stderr. diff --git a/.codex/skills/nemo-gym-reward-profiling/SKILL.md b/.codex/skills/nemo-gym-reward-profiling/SKILL.md index 89e99f1f5e..ebcb7758db 100644 --- a/.codex/skills/nemo-gym-reward-profiling/SKILL.md +++ b/.codex/skills/nemo-gym-reward-profiling/SKILL.md @@ -2,7 +2,7 @@ name: nemo-gym-reward-profiling description: >- Use to help users get started with Nemo Gym reward profiling. Covers the basic - ng_run, ng_collect_rollouts, and ng_reward_profile workflow, repeated rollouts, + gym env start, gym eval run, and gym eval profile workflow, repeated rollouts, materialized inputs, rollout JSONL artifacts, task and rollout identity, output inspection, partial profiling, and rollout_infos. For failed jobs, prefer nemo-gym-debugging. @@ -14,16 +14,16 @@ description: >- Use this skill when the user wants to run, understand, or lightly modify Nemo Gym reward profiling. Keep the answer oriented around the normal workflow: -`ng_run` starts model/resource servers, `ng_collect_rollouts` writes rollout artifacts, and `ng_reward_profile` generates profiling output from those artifacts. +`gym env start` starts model/resources servers, `gym eval run --no-serve` writes rollout artifacts, and `gym eval profile` generates profiling output from those artifacts. -If the user is primarily debugging a failed job or stack trace, use `$nemo-gym-debugging` first. +If the user is primarily debugging a failed job or stack trace, use the `nemo-gym-debugging` skill first. ## Basic Workflow 1. Identify the environment config paths and input JSONL. -2. Start Gym servers with `ng_run`. -3. Collect rollouts with `ng_collect_rollouts`; this writes `rollouts.jsonl` and `*_materialized_inputs.jsonl`. -4. Run `ng_reward_profile` on the materialized inputs and rollout JSONL to generate `*_reward_profiling.jsonl`. +2. Start Gym servers with `gym env start`. +3. Collect rollouts with `gym eval run --no-serve`; this writes `rollouts.jsonl` and `*_materialized_inputs.jsonl`. +4. Run `gym eval profile` on the materialized inputs and rollout JSONL to generate `*_reward_profiling.jsonl`. 5. Inspect line counts and profile rows. Repeated rollouts are the main profiling lever. `num_repeats=1` is valid, but per-task averages and variance are only meaningful with multiple rollouts per task. @@ -48,6 +48,6 @@ Load references only when the user needs that detail: ## Practical Defaults -- Treat `ng_reward_profile` as the reward profiling step; rollout collection does not write reward profile files. +- Treat `gym eval profile` as the reward profiling step; rollout collection does not write reward profile files. - Run strict profiling by default. If rollout collection stopped early, use `++allow_partial_rollouts=True` to profile completed rollouts and drop original input rows with no completed rollout. - Trust the target checkout's CLI help and `nemo_gym/reward_profile.py` over memory if flags differ. diff --git a/.codex/skills/nemo-gym-reward-profiling/references/output-format.md b/.codex/skills/nemo-gym-reward-profiling/references/output-format.md index 71ab7b572d..23ab8dbfd3 100644 --- a/.codex/skills/nemo-gym-reward-profiling/references/output-format.md +++ b/.codex/skills/nemo-gym-reward-profiling/references/output-format.md @@ -6,7 +6,7 @@ Reward profiling is built around joining materialized inputs with rollout result - `*_materialized_inputs.jsonl`: expanded inputs after repeat expansion. Each row should have `_ng_task_index` and `_ng_rollout_index`. - `rollouts.jsonl`: completed rollout results. Each row should have matching `_ng_task_index` and `_ng_rollout_index`. -- `*_reward_profiling.jsonl`: task-level summaries produced by `ng_reward_profile`. +- `*_reward_profiling.jsonl`: task-level summaries produced by `gym eval profile`. - `*_agent_metrics.json`: agent/global aggregate metrics. ## Task Profile Rows @@ -36,7 +36,7 @@ Full model responses stay in `rollouts.jsonl`. Join back to full rows with `(_ng ## Partial Profiles -Strict profiling is the default. If materialized inputs and rollout results do not have the same rollout keys, `ng_reward_profile` fails and suggests: +Strict profiling is the default. If materialized inputs and rollout results do not have the same rollout keys, `gym eval profile` fails and suggests: ```bash ++allow_partial_rollouts=True diff --git a/.codex/skills/nemo-gym-reward-profiling/references/quick-start.md b/.codex/skills/nemo-gym-reward-profiling/references/quick-start.md index 0e2c80838b..3b19d2e832 100644 --- a/.codex/skills/nemo-gym-reward-profiling/references/quick-start.md +++ b/.codex/skills/nemo-gym-reward-profiling/references/quick-start.md @@ -5,8 +5,6 @@ Substitute environment-specific config paths, input data, model endpoint, and ou ## Minimal Flow ```bash -CONFIG_PATHS="your_model_config_paths,your_env_config_paths" - POLICY_MODEL_NAME="your_policy_model_name" POLICY_BASE_URL="your_policy_base_url" POLICY_ENDPOINT_KEY="your_policy_endpoint_key" @@ -19,41 +17,43 @@ AGENT_NAME="your_agent_name" NUM_REPEATS=2 NUM_SAMPLES_IN_PARALLEL=8 -ng_run "+config_paths=[$CONFIG_PATHS]" \ - +policy_model_name="$POLICY_MODEL_NAME" \ - +policy_base_url="$POLICY_BASE_URL" \ - +policy_api_key="$POLICY_ENDPOINT_KEY" & +gym env start \ + --config your_model_config_paths \ + --config your_env_config_paths \ + --model "$POLICY_MODEL_NAME" \ + --model-url "$POLICY_BASE_URL" \ + --model-api-key "$POLICY_ENDPOINT_KEY" & NG_RUN_PID=$! trap 'kill "$NG_RUN_PID" 2>/dev/null || true' EXIT ./scripts/wait_for_servers.sh "$NG_RUN_PID" agent_args=() if [[ -n "$AGENT_NAME" ]]; then - agent_args=(+agent_name="$AGENT_NAME") + agent_args=(--agent "$AGENT_NAME") fi -ng_collect_rollouts \ +gym eval run --no-serve \ "${agent_args[@]}" \ - +input_jsonl_fpath="$DATA_JSONL" \ - +output_jsonl_fpath="$ROLLOUTS_JSONL" \ - +num_repeats="$NUM_REPEATS" \ - +num_samples_in_parallel="$NUM_SAMPLES_IN_PARALLEL" - -ng_reward_profile \ - ++materialized_inputs_jsonl_fpath="$MATERIALIZED_JSONL" \ - ++rollouts_jsonl_fpath="$ROLLOUTS_JSONL" + --input "$DATA_JSONL" \ + --output "$ROLLOUTS_JSONL" \ + --num-repeats "$NUM_REPEATS" \ + --concurrency "$NUM_SAMPLES_IN_PARALLEL" + +gym eval profile \ + --inputs "$MATERIALIZED_JSONL" \ + --rollouts "$ROLLOUTS_JSONL" ``` If rows already contain `agent_ref`, leave `AGENT_NAME` empty. Passing `+agent_name` supplies a default for rows without one. ## Partial Rollouts -By default, `ng_reward_profile` expects every materialized input row to have a matching rollout row. If collection stopped early, profile the completed rollouts with: +By default, `gym eval profile` expects every materialized input row to have a matching rollout row. If collection stopped early, profile the completed rollouts with: ```bash -ng_reward_profile \ - ++materialized_inputs_jsonl_fpath="$MATERIALIZED_JSONL" \ - ++rollouts_jsonl_fpath="$ROLLOUTS_JSONL" \ +gym eval profile \ + --inputs "$MATERIALIZED_JSONL" \ + --rollouts "$ROLLOUTS_JSONL" \ ++allow_partial_rollouts=True ``` diff --git a/CLAUDE.md b/CLAUDE.md index d14cab2cf4..2372b4fc86 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,14 +82,16 @@ uv venv && uv sync --extra dev --group docs pre-commit install # Run servers -ng_run "+config_paths=[resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --resources-server example_single_tool_call \ + --model-type vllm_model # Run tests for a specific server (creates .venv per server, installs deps, runs pytest) # First run is slow. Use skip_venv_if_present config or place a .venv to skip venv creation. -ng_test +entrypoint=resources_servers/example_single_tool_call +gym env test --resources-server example_single_tool_call # Run all server tests -ng_test_all +gym env test # Run core library unit tests pytest tests/unit_tests/ -x @@ -105,13 +107,13 @@ ruff format . pre-commit run --all-files # Check server health -ng_status +gym env status -# Dev test (runs pytest directly in server dir, no venv isolation) -ng_dev_test +entrypoint=resources_servers/example_single_tool_call +# Dev test (runs the core unit tests with coverage: pytest --cov) +gym dev test # Dump merged config -ng_dump_config "+config_paths=[...]" +gym env resolve --config ... ``` ## Code Style @@ -143,4 +145,4 @@ git checkout -- resources_servers/other_server/ ## Cluster / HPC Gotchas - **Ray socket path length**: On systems with long working directory paths (e.g. Lustre mounts), Ray's AF_UNIX socket paths can exceed the 107-byte Linux limit. Fix: `RAY_TMPDIR=/tmp` before running tests or `ray.init()`. -- **`ng_test` venv isolation**: `ng_test` creates isolated venvs per resources server. `os.environ` changes in Python don't propagate — set env vars externally (e.g. `RAY_TMPDIR=/tmp ng_test ...`). +- **`gym env test` venv isolation**: `gym env test` creates isolated venvs per resources server. `os.environ` changes in Python don't propagate — set env vars externally (e.g. `RAY_TMPDIR=/tmp gym env test ...`). diff --git a/README.md b/README.md index 45f5cf29c9..150f42a6ab 100644 --- a/README.md +++ b/README.md @@ -100,10 +100,9 @@ Run your agent on a set of tasks and score the results. This example uses a simp NeMo Gym uses local servers to coordinate your model, agent, and task verification. Start them first: ```bash -environment_config="resources_servers/mcqa/configs/mcqa.yaml" -model_config="responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[${environment_config},${model_config}]" +gym env start \ + --resources-server mcqa \ + --model-type openai_model ``` You should see three server instances starting: @@ -121,12 +120,12 @@ In a new terminal, run your agent on a single task to verify everything works: ```bash source .venv/bin/activate -ng_collect_rollouts \ - +agent_name=mcqa_simple_agent \ - +input_jsonl_fpath=resources_servers/mcqa/data/example.jsonl \ - +output_jsonl_fpath=results/mcqa_rollouts.jsonl \ - +limit=5 \ - +num_repeats=1 +gym eval run --no-serve \ + --agent mcqa_simple_agent \ + --input resources_servers/mcqa/data/example.jsonl \ + --output results/mcqa_rollouts.jsonl \ + --limit 5 \ + --num-repeats 1 ``` You should see a progress bar followed by aggregate metrics: @@ -146,7 +145,7 @@ Rollouts: results/mcqa_rollouts.jsonl Aggregate metrics: results/mcqa_rollouts_aggregate_metrics.json ``` -For per-task pass rates, see the [`ng_reward_profile`](https://docs.nvidia.com/nemo/gym/main/reference/cli-commands) command. +For per-task pass rates, see the [`gym eval profile`](https://docs.nvidia.com/nemo/gym/main/reference/cli-commands) command. ### Next Steps diff --git a/benchmarks/aalcr/README.md b/benchmarks/aalcr/README.md index 52a0d823d0..996e24f4fa 100644 --- a/benchmarks/aalcr/README.md +++ b/benchmarks/aalcr/README.md @@ -1,15 +1,13 @@ # Prepare data ```bash -config_paths="benchmarks/aalcr/config.yaml" -ng_prepare_benchmark "+config_paths=[$config_paths]" +gym eval prepare --benchmark aalcr ``` # Run ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/aalcr/config.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ +gym eval run \ + --model-type vllm_model \ + --benchmark aalcr \ ++output_jsonl_fpath=results/benchmarks/aalcr.jsonl \ ++overwrite_metrics_conflicts=true \ ++split=benchmark \ diff --git a/benchmarks/aime24-x/README.md b/benchmarks/aime24-x/README.md index e59d3bfde3..7fbe707469 100644 --- a/benchmarks/aime24-x/README.md +++ b/benchmarks/aime24-x/README.md @@ -21,7 +21,7 @@ This benchmark reuses `math_with_judge` in symbolic-only mode ## Data Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/aime24-x/config.yaml]" +gym eval prepare --benchmark aime24-x ``` That writes `benchmarks/aime24-x/data/aime24-x_benchmark.jsonl`. @@ -36,16 +36,21 @@ python benchmarks/aime24-x/prepare.py --prompt_language en ## Quickstart ```bash -ng_run "+config_paths=[benchmarks/aime24-x/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --benchmark aime24-x \ + --model-type vllm_model ``` Then in another shell: ```bash -ng_collect_rollouts \ - +agent_name=aime24-x_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/aime24-x/data/aime24-x_benchmark.jsonl \ - +output_jsonl_fpath=results/aime24-x/rollouts.jsonl \ - +num_repeats=32 +num_repeats_add_seed=true \ - "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 65536}" +gym eval run --no-serve \ + --agent aime24-x_math_with_judge_simple_agent \ + --input benchmarks/aime24-x/data/aime24-x_benchmark.jsonl \ + --output results/aime24-x/rollouts.jsonl \ + --num-repeats 32 \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 65536 \ + +num_repeats_add_seed=true ``` diff --git a/benchmarks/aime25-x/README.md b/benchmarks/aime25-x/README.md index 1ac1453ff1..23f145180d 100644 --- a/benchmarks/aime25-x/README.md +++ b/benchmarks/aime25-x/README.md @@ -21,7 +21,7 @@ This benchmark reuses `math_with_judge` in symbolic-only mode ## Data Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/aime25-x/config.yaml]" +gym eval prepare --benchmark aime25-x ``` That writes `benchmarks/aime25-x/data/aime25-x_benchmark.jsonl`. @@ -36,17 +36,22 @@ python benchmarks/aime25-x/prepare.py --prompt_language en ## Quickstart ```bash -ng_run "+config_paths=[benchmarks/aime25-x/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --benchmark aime25-x \ + --model-type vllm_model ``` Then in another shell: ```bash -ng_collect_rollouts \ - +agent_name=aime25-x_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/aime25-x/data/aime25-x_benchmark.jsonl \ - +output_jsonl_fpath=results/aime25-x/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml \ - +num_repeats=32 +num_repeats_add_seed=true \ - "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 65536}" +gym eval run --no-serve \ + --agent aime25-x_math_with_judge_simple_agent \ + --input benchmarks/aime25-x/data/aime25-x_benchmark.jsonl \ + --output results/aime25-x/rollouts.jsonl \ + --num-repeats 32 \ + --prompt-config benchmarks/prompts/generic/default.yaml \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 65536 \ + +num_repeats_add_seed=true ``` diff --git a/benchmarks/aime26/README.md b/benchmarks/aime26/README.md index a7d0c401d6..8013220481 100644 --- a/benchmarks/aime26/README.md +++ b/benchmarks/aime26/README.md @@ -5,22 +5,23 @@ AIME 2026 (American Invitational Mathematics Examination) — 30 competition mat ## Prepare data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/aime26/config.yaml]" +gym eval prepare --benchmark aime26 ``` ## Run servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/aime26/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark aime26 ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=aime26_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/aime26/data/aime26_benchmark.jsonl \ - +output_jsonl_fpath=results/aime26_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent aime26_math_with_judge_simple_agent \ + --input benchmarks/aime26/data/aime26_benchmark.jsonl \ + --output results/aime26_rollouts.jsonl \ + --num-repeats 4 ``` diff --git a/benchmarks/answer-judge/README.md b/benchmarks/answer-judge/README.md index 3313ae000c..6418783221 100644 --- a/benchmarks/answer-judge/README.md +++ b/benchmarks/answer-judge/README.md @@ -13,17 +13,17 @@ here is the same deterministic `Judgement: Yes/No` parsing used by Skills' ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/answer-judge/config.yaml]" +gym eval prepare --benchmark answer-judge # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/answer-judge/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark answer-judge # Collecting rollouts -ng_collect_rollouts \ - +agent_name=answer_judge_math_proof_judgement_simple_agent \ - +input_jsonl_fpath=benchmarks/answer-judge/data/answer-judge_benchmark.jsonl \ - +output_jsonl_fpath=results/answer-judge/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/judge/math.yaml +gym eval run --no-serve \ + --agent answer_judge_math_proof_judgement_simple_agent \ + --input benchmarks/answer-judge/data/answer-judge_benchmark.jsonl \ + --output results/answer-judge/rollouts.jsonl \ + --prompt-config benchmarks/prompts/judge/math.yaml ``` diff --git a/benchmarks/apex_shortlist/README.md b/benchmarks/apex_shortlist/README.md index 2ced13a9f8..bf41621ec1 100644 --- a/benchmarks/apex_shortlist/README.md +++ b/benchmarks/apex_shortlist/README.md @@ -26,7 +26,7 @@ Solve the following math problem. Make sure to put the answer (and only answer) ## Data preparation ```bash -ng_prepare_benchmark '+config_paths=[benchmarks/apex_shortlist/config.yaml]' +gym eval prepare --benchmark apex_shortlist ``` Writes `data/apex_shortlist_benchmark.jsonl` with one row per problem: @@ -35,17 +35,17 @@ Writes `data/apex_shortlist_benchmark.jsonl` with one row per problem: ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/apex_shortlist/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark apex_shortlist ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=apex_shortlist_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/apex_shortlist/data/apex_shortlist_benchmark.jsonl \ - +output_jsonl_fpath=results/apex_shortlist_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent apex_shortlist_math_with_judge_simple_agent \ + --input benchmarks/apex_shortlist/data/apex_shortlist_benchmark.jsonl \ + --output results/apex_shortlist_rollouts.jsonl \ + --num-repeats 4 ``` diff --git a/benchmarks/arena_hard/README.md b/benchmarks/arena_hard/README.md index 55e75997b5..d6038ce450 100644 --- a/benchmarks/arena_hard/README.md +++ b/benchmarks/arena_hard/README.md @@ -15,7 +15,7 @@ for the judging protocol and metric details. ## Data Runtime download only — benchmark JSONL is not committed. Run -[`prepare.py`](prepare.py) (or `ng_prepare_benchmark`) to populate +[`prepare.py`](prepare.py) (or `gym eval prepare`) to populate `data/arena_hard_benchmark.jsonl`. The prepare script fetches questions and the baseline directly from the arena-hard-auto GitHub repo, joins by `uid`, and emits one row per question with `question`, @@ -28,20 +28,20 @@ to pick the standard judge prompt. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/arena_hard/config.yaml]" +gym eval prepare --benchmark arena_hard # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/arena_hard/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark arena_hard # Collecting rollouts -ng_collect_rollouts \ - +agent_name=arena_hard_arena_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/arena_hard/data/arena_hard_benchmark.jsonl \ - +output_jsonl_fpath=results/arena_hard_rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent arena_hard_arena_judge_simple_agent \ + --input benchmarks/arena_hard/data/arena_hard_benchmark.jsonl \ + --output results/arena_hard_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/prompts/generic/default.yaml ``` ## Metrics diff --git a/benchmarks/arena_hard_v2/README.md b/benchmarks/arena_hard_v2/README.md index e88a228525..5a8b160a25 100644 --- a/benchmarks/arena_hard_v2/README.md +++ b/benchmarks/arena_hard_v2/README.md @@ -21,7 +21,7 @@ for the judging protocol and metric details. ## Data Runtime download only — benchmark JSONL is not committed. Run -[`prepare.py`](prepare.py) (or `ng_prepare_benchmark`) to populate +[`prepare.py`](prepare.py) (or `gym eval prepare`) to populate `data/arena_hard_v2_benchmark.jsonl`. The prepare script fetches questions and both baselines directly from the arena-hard-auto GitHub repo, joins by `uid`, and emits one row per question with `question`, @@ -31,19 +31,19 @@ repo, joins by `uid`, and emits one row per question with `question`, ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/arena_hard_v2/config.yaml]" +gym eval prepare --benchmark arena_hard_v2 # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/arena_hard_v2/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark arena_hard_v2 # Collecting rollouts -ng_collect_rollouts \ - +agent_name=arena_hard_v2_arena_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/arena_hard_v2/data/arena_hard_v2_benchmark.jsonl \ - +output_jsonl_fpath=results/arena_hard_v2_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent arena_hard_v2_arena_judge_simple_agent \ + --input benchmarks/arena_hard_v2/data/arena_hard_v2_benchmark.jsonl \ + --output results/arena_hard_v2_rollouts.jsonl \ + --num-repeats 4 ``` ## Metrics diff --git a/benchmarks/asr_leaderboard/README.md b/benchmarks/asr_leaderboard/README.md index 7326651f22..ce872ff0e9 100644 --- a/benchmarks/asr_leaderboard/README.md +++ b/benchmarks/asr_leaderboard/README.md @@ -23,7 +23,7 @@ System + user templates live in [`prompts/default.yaml`](prompts/default.yaml). ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/asr_leaderboard/config.yaml]" +gym eval prepare --benchmark asr_leaderboard ``` Downloads the 8 ESB subsets (~tens of GB of FLAC) and writes @@ -32,18 +32,18 @@ Downloads the 8 ESB subsets (~tens of GB of FLAC) and writes ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/asr_leaderboard/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark asr_leaderboard ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=asr_leaderboard_asr_with_pc_simple_agent \ - +output_jsonl_fpath=results/asr_leaderboard_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent asr_leaderboard_asr_with_pc_simple_agent \ + --output results/asr_leaderboard_rollouts.jsonl \ + --num-repeats 1 ``` ## Verification diff --git a/benchmarks/bigcodebench/README.md b/benchmarks/bigcodebench/README.md index 0722b2eaa9..b638f3a0db 100644 --- a/benchmarks/bigcodebench/README.md +++ b/benchmarks/bigcodebench/README.md @@ -13,27 +13,27 @@ the `full` split (~1140 problems) is `bigcode/bigcodebench@v0.1.4`. ```bash # Prepare benchmark data (hard split, ~148 problems) -ng_prepare_benchmark "+config_paths=[benchmarks/bigcodebench/config.yaml]" +gym eval prepare --benchmark bigcodebench # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/bigcodebench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark bigcodebench # Collecting rollouts (5-row smoke test against the baked example set) -ng_collect_rollouts \ - +agent_name=bigcodebench_benchmark_simple_agent \ - +input_jsonl_fpath=resources_servers/bigcodebench/data/example.jsonl \ - +output_jsonl_fpath=results/bigcodebench_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent bigcodebench_benchmark_simple_agent \ + --input resources_servers/bigcodebench/data/example.jsonl \ + --output results/bigcodebench_rollouts.jsonl \ + --num-repeats 1 ``` -The benchmark JSONL written by ``ng_prepare_benchmark`` is unbaked +The benchmark JSONL written by ``gym eval prepare`` is unbaked (rows have ``question`` + ``verifier_metadata``; the prompt template is -applied by the agent at ``/run`` time). Standalone ``ng_collect_rollouts`` +applied by the agent at ``/run`` time). Standalone ``gym eval run --no-serve`` expects pre-baked ``responses_create_params.input``, so for full-dataset runs use the production orchestrator (``nemo_gym_rollouts`` from -NeMo-Skills) rather than ``ng_collect_rollouts`` directly. +NeMo-Skills) rather than ``gym eval run --no-serve`` directly. `prepare.py` exposes a `--split` flag (`hard` or `full`); the config defaults to `hard` to match the recipe's parity-comparison run. diff --git a/benchmarks/birdbench/README.md b/benchmarks/birdbench/README.md index ba7708c352..872193bb28 100644 --- a/benchmarks/birdbench/README.md +++ b/benchmarks/birdbench/README.md @@ -10,7 +10,7 @@ Execution-based text-to-SQL on BIRD dev, bound to the `bird_sql` resource server ## Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/birdbench/config.yaml]" +gym eval prepare --benchmark birdbench ``` This downloads the BIRD `dev.zip` (≈1.4 GB) via @@ -22,9 +22,9 @@ and writes `data/birdbench_benchmark.jsonl`. Each row has ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/birdbench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark birdbench ``` Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in @@ -33,11 +33,11 @@ Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=birdbench_bird_sql_simple_agent \ - +input_jsonl_fpath=benchmarks/birdbench/data/birdbench_benchmark.jsonl \ - +output_jsonl_fpath=results/birdbench_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent birdbench_bird_sql_simple_agent \ + --input benchmarks/birdbench/data/birdbench_benchmark.jsonl \ + --output results/birdbench_rollouts.jsonl \ + --num-repeats 4 ``` For a 5-example smoke test against the resource server's `example.jsonl`, diff --git a/benchmarks/browsecomp/README.md b/benchmarks/browsecomp/README.md index 7d7d2ef9dd..4d28ef61c1 100644 --- a/benchmarks/browsecomp/README.md +++ b/benchmarks/browsecomp/README.md @@ -15,23 +15,21 @@ Qwen3-235B-A22B-Instruct-2507-FP8: 2. Prepare the benchmark dataset ```bash -config_paths="benchmarks/browsecomp/config.yaml" -ng_prepare_benchmark "+config_paths=[$config_paths]" +gym eval prepare --benchmark browsecomp ``` 3. Run the benchmark against a VLLMModel ```bash WANDB_PROJECT= EXPERIMENT_NAME= -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/browsecomp/config.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ +gym eval run \ + --model-type vllm_model \ + --benchmark browsecomp \ + --output results/$EXPERIMENT_NAME.jsonl \ + --split benchmark \ + --model-url ??? \ + --model-api-key ??? \ + --model ??? \ +wandb_project=$WANDB_PROJECT \ - +wandb_name=$EXPERIMENT_NAME \ - ++output_jsonl_fpath=results/$EXPERIMENT_NAME.jsonl \ - ++split=benchmark \ - ++policy_base_url=??? \ - ++policy_api_key=??? \ - ++policy_model_name=??? + +wandb_name=$EXPERIMENT_NAME ``` diff --git a/benchmarks/finance_sec_search/README.md b/benchmarks/finance_sec_search/README.md index 5a43b6c9fb..28b6a2c7dc 100644 --- a/benchmarks/finance_sec_search/README.md +++ b/benchmarks/finance_sec_search/README.md @@ -27,13 +27,13 @@ server's `/prompt_templates`. Without web search: ```bash -ng_prepare_benchmark '+config_paths=[benchmarks/finance_sec_search/config_no_web_search.yaml]' +gym eval prepare --benchmark finance_sec_search/config_no_web_search ``` With web search (requires `tavily_api_key` in `env.yaml`): ```bash -ng_prepare_benchmark '+config_paths=[benchmarks/finance_sec_search/config_web_search.yaml]' +gym eval prepare --benchmark finance_sec_search/config_web_search ``` Downloads `public.csv` from the Vals AI GitHub repo and writes benchmark @@ -47,16 +47,16 @@ JSONL to `data/`. ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/finance_sec_search/config_no_web_search.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark finance_sec_search/config_no_web_search ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=finance_sec_search_benchmark_agent \ - +input_jsonl_fpath=benchmarks/finance_sec_search/data/finance_sec_search_benchmark.jsonl \ - +output_jsonl_fpath=results/finance_sec_search_rollouts.jsonl +gym eval run --no-serve \ + --agent finance_sec_search_benchmark_agent \ + --input benchmarks/finance_sec_search/data/finance_sec_search_benchmark.jsonl \ + --output results/finance_sec_search_rollouts.jsonl ``` diff --git a/benchmarks/finance_sec_search/prepare.py b/benchmarks/finance_sec_search/prepare.py index 2a6268238d..c040dbd8e6 100644 --- a/benchmarks/finance_sec_search/prepare.py +++ b/benchmarks/finance_sec_search/prepare.py @@ -228,7 +228,7 @@ def prepare(include_web_search: bool = False) -> Path: Args: include_web_search: Include the web_search tool in tool definitions. - Default False (ng_prepare_benchmark calls prepare() with no args). + Default False (gym eval prepare calls prepare() with no args). Pass True or use --include-web-search on the CLI. """ DATA_DIR.mkdir(parents=True, exist_ok=True) diff --git a/benchmarks/finance_sec_search/prepare_web_search.py b/benchmarks/finance_sec_search/prepare_web_search.py index 491a430cff..b0680cfea4 100644 --- a/benchmarks/finance_sec_search/prepare_web_search.py +++ b/benchmarks/finance_sec_search/prepare_web_search.py @@ -15,7 +15,7 @@ """Prepare finance_sec_search benchmark with web_search tool included. Thin wrapper around prepare.prepare() — used by config_web_search.yaml -so ng_prepare_benchmark produces the web_search variant. +so gym eval prepare produces the web_search variant. """ from pathlib import Path diff --git a/benchmarks/flores200/README.md b/benchmarks/flores200/README.md index 8f770cbedd..df3ef4872b 100644 --- a/benchmarks/flores200/README.md +++ b/benchmarks/flores200/README.md @@ -18,7 +18,7 @@ the Ray GPU-scheduled COMET path. ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/flores200/config.yaml]" +gym eval prepare --benchmark flores200 ``` Writes `data/flores200_devtest_benchmark.jsonl` and pre-fetches the @@ -41,21 +41,21 @@ only advertised on multi-node SLURM deployments via NeMo-Skills' override and rely on corpus-BLEU only: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/flores200/config.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --model-type vllm_model \ + --benchmark flores200 \ "++flores200_wmt_translation_resources_server.resources_servers.wmt_translation.compute_comet=false" ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=flores200_wmt_translation_simple_agent \ - +prompt_config=benchmarks/flores200/prompts/default.yaml \ - +input_jsonl_fpath=benchmarks/flores200/data/flores200_devtest_benchmark.jsonl \ - +output_jsonl_fpath=results/flores200_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent flores200_wmt_translation_simple_agent \ + --input benchmarks/flores200/data/flores200_devtest_benchmark.jsonl \ + --output results/flores200_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/flores200/prompts/default.yaml ``` ## End-to-end on a SLURM cluster (via NeMo-Skills) @@ -76,7 +76,7 @@ ns run_cmd \ --cluster \ --container nemo-gym \ --expname flores200_prepare \ - --command 'ng_prepare_benchmark "+config_paths=[benchmarks/flores200/config.yaml]"' + --command 'gym eval prepare --benchmark flores200' ``` This populates `benchmarks/flores200/data/flores200_devtest_benchmark.jsonl` diff --git a/benchmarks/frontierscience_olympiad/README.md b/benchmarks/frontierscience_olympiad/README.md index 645a646e64..da7e141641 100644 --- a/benchmarks/frontierscience_olympiad/README.md +++ b/benchmarks/frontierscience_olympiad/README.md @@ -36,20 +36,20 @@ example, the original Skills configuration uses `o3-mini-2025-01-31` via ```bash # Prepare benchmark data (downloads from HuggingFace) -ng_prepare_benchmark "+config_paths=[benchmarks/frontierscience_olympiad/config.yaml]" +gym eval prepare --benchmark frontierscience_olympiad # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/frontierscience_olympiad/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark frontierscience_olympiad # Collecting rollouts (4 rollouts per task, matching Skills' default) -ng_collect_rollouts \ - +agent_name=frontierscience_olympiad_frontierscience_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/frontierscience_olympiad/data/frontierscience_olympiad_benchmark.jsonl \ - +prompt_config=benchmarks/frontierscience_olympiad/prompts/default.yaml \ - +output_jsonl_fpath=results/frontierscience_olympiad_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent frontierscience_olympiad_frontierscience_judge_simple_agent \ + --input benchmarks/frontierscience_olympiad/data/frontierscience_olympiad_benchmark.jsonl \ + --prompt-config benchmarks/frontierscience_olympiad/prompts/default.yaml \ + --output results/frontierscience_olympiad_rollouts.jsonl \ + --num-repeats 4 ``` For Nemotron-3-Nano and other reasoning models, start vLLM with diff --git a/benchmarks/gdpval/README.md b/benchmarks/gdpval/README.md index b2e50937f8..e1029ebbaa 100644 --- a/benchmarks/gdpval/README.md +++ b/benchmarks/gdpval/README.md @@ -11,7 +11,7 @@ Downloads `openai/gdpval` from HuggingFace and writes `data/gdpval_benchmark.jsonl`: ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/gdpval/config.yaml]" +gym eval prepare --benchmark gdpval ``` ## Run rubric mode (default) @@ -19,15 +19,14 @@ ng_prepare_benchmark "+config_paths=[benchmarks/gdpval/config.yaml]" Each deliverable is scored 0–1 against the task rubric. ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/gdpval/config.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++output_jsonl_fpath=results/gdpval_rubric.jsonl \ - ++split=benchmark \ - ++policy_base_url= \ - ++policy_api_key= \ - ++policy_model_name= +gym eval run \ + --model-type vllm_model \ + --benchmark gdpval \ + --output results/gdpval_rubric.jsonl \ + --split benchmark \ + --model-url \ + --model-api-key \ + --model ``` Required environment variables for the judge: @@ -45,10 +44,11 @@ same `task_id`; aggregate metrics include ELO relative to a configurable anchor (default 1000). ```bash -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++output_jsonl_fpath=results/gdpval_compare.jsonl \ - ++split=benchmark \ +gym eval run \ + --model-type vllm_model \ + --benchmark gdpval \ + --output results/gdpval_compare.jsonl \ + --split benchmark \ ++gdpval_resources_server.resources_servers.gdpval.reward_mode=comparison \ ++gdpval_resources_server.resources_servers.gdpval.reference_deliverables_dir=/path/to/reference/output ``` @@ -59,7 +59,7 @@ the deliverable files (the same layout the Stirrup agent persists). ## Aggregate metrics -After `ng_e2e_collect_rollouts` returns, the resources server's +After `gym eval run` returns, the resources server's `/aggregate_metrics` endpoint emits headline scores in `results/_metrics.json`: diff --git a/benchmarks/global-piqa/README.md b/benchmarks/global-piqa/README.md index 73516ff902..110434521e 100644 --- a/benchmarks/global-piqa/README.md +++ b/benchmarks/global-piqa/README.md @@ -16,17 +16,17 @@ server. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/global-piqa/config.yaml]" +gym eval prepare --benchmark global-piqa # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/global-piqa/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark global-piqa # Collecting rollouts -ng_collect_rollouts \ - +agent_name=global_piqa_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/global-piqa/data/global-piqa_benchmark.jsonl \ - +output_jsonl_fpath=results/global-piqa/rollouts.jsonl \ - +prompt_config=benchmarks/global-piqa/prompts/default.yaml +gym eval run --no-serve \ + --agent global_piqa_mcqa_simple_agent \ + --input benchmarks/global-piqa/data/global-piqa_benchmark.jsonl \ + --output results/global-piqa/rollouts.jsonl \ + --prompt-config benchmarks/global-piqa/prompts/default.yaml ``` diff --git a/benchmarks/gpqa-x/README.md b/benchmarks/gpqa-x/README.md index f50c9a8a69..e17e2db729 100644 --- a/benchmarks/gpqa-x/README.md +++ b/benchmarks/gpqa-x/README.md @@ -22,7 +22,7 @@ This benchmark reuses the `mcqa` resource server, which matches Skills' ## Data Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/gpqa-x/config.yaml]" +gym eval prepare --benchmark gpqa-x ``` That writes `benchmarks/gpqa-x/data/gpqa-x_benchmark.jsonl`. @@ -37,17 +37,22 @@ python benchmarks/gpqa-x/prepare.py --prompt_language en ## Quickstart ```bash -ng_run "+config_paths=[benchmarks/gpqa-x/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --benchmark gpqa-x \ + --model-type vllm_model ``` Then in another shell: ```bash -ng_collect_rollouts \ - +agent_name=gpqa-x_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/gpqa-x/data/gpqa-x_benchmark.jsonl \ - +output_jsonl_fpath=results/gpqa-x/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml \ - +num_repeats=8 +num_repeats_add_seed=true \ - "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 32768}" +gym eval run --no-serve \ + --agent gpqa-x_mcqa_simple_agent \ + --input benchmarks/gpqa-x/data/gpqa-x_benchmark.jsonl \ + --output results/gpqa-x/rollouts.jsonl \ + --prompt-config benchmarks/prompts/generic/default.yaml \ + --num-repeats 8 \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 32768 \ + +num_repeats_add_seed=true ``` diff --git a/benchmarks/gpqa/README.md b/benchmarks/gpqa/README.md index 1801de0c52..49f61c8073 100644 --- a/benchmarks/gpqa/README.md +++ b/benchmarks/gpqa/README.md @@ -13,13 +13,16 @@ This benchmark uses the `mcqa` resource server with the `mcqa_simple_agent`. ```bash # Prepare data -ng_prepare_benchmark "+config_paths=[benchmarks/gpqa/config.yaml]" +gym eval prepare --benchmark gpqa # Start servers -ng_run "+config_paths=[benchmarks/gpqa/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --benchmark gpqa \ + --model-type vllm_model # Collect rollouts -ng_collect_rollouts \ - "+config_paths=[benchmarks/gpqa/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +output_jsonl_fpath=results/gpqa.jsonl +gym eval run --no-serve \ + --benchmark gpqa \ + --model-type vllm_model \ + --output results/gpqa.jsonl ``` diff --git a/benchmarks/gpqa/prepare.py b/benchmarks/gpqa/prepare.py index 045732aff8..1dcf246dc9 100644 --- a/benchmarks/gpqa/prepare.py +++ b/benchmarks/gpqa/prepare.py @@ -19,7 +19,7 @@ compatible with the mcqa resource server. Output is raw data (no prompts baked in). Use prompt_config at rollout time -to specify the prompt, or ng_materialize_prompts to produce RL-ready data. +to specify the prompt, or gym dataset render to produce RL-ready data. """ import hashlib diff --git a/benchmarks/graphwalks/README.md b/benchmarks/graphwalks/README.md index 526a6e9152..271864987d 100644 --- a/benchmarks/graphwalks/README.md +++ b/benchmarks/graphwalks/README.md @@ -28,10 +28,10 @@ The N3 1M variant requires HF auth for the gated NVIDIA repo ```bash # Default (o200k_base, no filter) -ng_prepare_benchmark "+config_paths=[benchmarks/graphwalks/config.yaml]" +gym eval prepare --benchmark graphwalks # N3 1M variant -ng_prepare_benchmark "+config_paths=[benchmarks/graphwalks/config_n3_1m.yaml]" +gym eval prepare --benchmark graphwalks/config_n3_1m ``` For one-off custom builds (different tokenizer / cap / output path), @@ -47,25 +47,27 @@ python benchmarks/graphwalks/prepare.py \ ## Start environment ```bash -ng_run "+config_paths=[benchmarks/graphwalks/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --benchmark graphwalks \ + --model-type vllm_model ``` ## Collect rollouts ```bash # Default variant -ng_collect_rollouts \ - +agent_name=graphwalks_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/graphwalks/data/graphwalks_benchmark.jsonl \ - +output_jsonl_fpath=results/graphwalks_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent graphwalks_benchmark_simple_agent \ + --input benchmarks/graphwalks/data/graphwalks_benchmark.jsonl \ + --output results/graphwalks_rollouts.jsonl \ + --num-repeats 4 # N3 1M variant -ng_collect_rollouts \ - +agent_name=graphwalks_n3_1m_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/graphwalks/data/graphwalks_n3_1m_benchmark.jsonl \ - +output_jsonl_fpath=results/graphwalks_n3_1m_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent graphwalks_n3_1m_benchmark_simple_agent \ + --input benchmarks/graphwalks/data/graphwalks_n3_1m_benchmark.jsonl \ + --output results/graphwalks_n3_1m_rollouts.jsonl \ + --num-repeats 4 ``` ## Metrics diff --git a/benchmarks/graphwalks/prepare.py b/benchmarks/graphwalks/prepare.py index cbbd963d8f..1aa5c88480 100644 --- a/benchmarks/graphwalks/prepare.py +++ b/benchmarks/graphwalks/prepare.py @@ -34,7 +34,7 @@ Invocation ---------- -``ng_prepare_benchmark`` calls ``prepare()`` with no arguments, using +``gym eval prepare`` calls ``prepare()`` with no arguments, using the defaults below. To build a custom variant, run this script directly:: diff --git a/benchmarks/gsm8k/README.md b/benchmarks/gsm8k/README.md index d9a425048d..4043b38e85 100644 --- a/benchmarks/gsm8k/README.md +++ b/benchmarks/gsm8k/README.md @@ -13,17 +13,17 @@ the expected answer is integer-valued), then renames `problem` -> ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/gsm8k/config.yaml]" +gym eval prepare --benchmark gsm8k # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/gsm8k/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark gsm8k # Collecting rollouts -ng_collect_rollouts \ - +agent_name=gsm8k_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/gsm8k/data/gsm8k_benchmark.jsonl \ - +output_jsonl_fpath=results/gsm8k_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent gsm8k_math_with_judge_simple_agent \ + --input benchmarks/gsm8k/data/gsm8k_benchmark.jsonl \ + --output results/gsm8k_rollouts.jsonl \ + --num-repeats 4 ``` diff --git a/benchmarks/hendrycks_math/README.md b/benchmarks/hendrycks_math/README.md index 1b51db38f5..ffee07f070 100644 --- a/benchmarks/hendrycks_math/README.md +++ b/benchmarks/hendrycks_math/README.md @@ -12,17 +12,17 @@ applies Skills' renames (`answer` -> `expected_answer`, `question` -> ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/hendrycks_math/config.yaml]" +gym eval prepare --benchmark hendrycks_math # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/hendrycks_math/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark hendrycks_math # Collecting rollouts -ng_collect_rollouts \ - +agent_name=hendrycks_math_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/hendrycks_math/data/hendrycks_math_benchmark.jsonl \ - +output_jsonl_fpath=results/hendrycks_math_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent hendrycks_math_math_with_judge_simple_agent \ + --input benchmarks/hendrycks_math/data/hendrycks_math_benchmark.jsonl \ + --output results/hendrycks_math_rollouts.jsonl \ + --num-repeats 4 ``` diff --git a/benchmarks/hle/README.md b/benchmarks/hle/README.md index 61d83fcb28..ad4cc5fef4 100644 --- a/benchmarks/hle/README.md +++ b/benchmarks/hle/README.md @@ -25,7 +25,7 @@ huggingface-cli login ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/hle/config.yaml]" +gym eval prepare --benchmark hle ``` Downloads `cais/hle`, filters to text-only questions, and writes @@ -34,9 +34,9 @@ Downloads `cais/hle`, filters to text-only questions, and writes ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/hle/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark hle ``` Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in @@ -45,13 +45,13 @@ Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=hle_equivalence_llm_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/hle/data/hle_benchmark.jsonl \ - +output_jsonl_fpath=results/hle_rollouts.jsonl \ - +prompt_config=benchmarks/hle/prompts/default.yaml \ - +num_repeats=1 \ - "++responses_create_params={temperature: 0.0}" +gym eval run --no-serve \ + --agent hle_equivalence_llm_judge_simple_agent \ + --input benchmarks/hle/data/hle_benchmark.jsonl \ + --output results/hle_rollouts.jsonl \ + --prompt-config benchmarks/hle/prompts/default.yaml \ + --num-repeats 1 \ + --temperature 0.0 ``` Use `temperature: 0.0` to match the nemo-skills evaluation setup and ensure reproducible scores. diff --git a/benchmarks/hmmt_feb25/README.md b/benchmarks/hmmt_feb25/README.md index 067dbbabb1..a0806e2af1 100644 --- a/benchmarks/hmmt_feb25/README.md +++ b/benchmarks/hmmt_feb25/README.md @@ -25,7 +25,7 @@ Solve the following math problem. Make sure to put the answer (and only answer) ## Data preparation ``` -ng_prepare_benchmark '+config_paths=[benchmarks/hmmt_feb25/config.yaml]' +gym eval prepare --benchmark hmmt_feb25 ``` Writes `data/hmmt_feb25_benchmark.jsonl` with one row per problem: @@ -37,7 +37,9 @@ Start the benchmark's servers (inherits `math_with_judge` in symbolic-only mode plus a vLLM model server — adjust the model config to match your deployment): ``` -ng_run "+config_paths=[benchmarks/hmmt_feb25/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --benchmark hmmt_feb25 \ + --model-type vllm_model ``` In a separate shell, collect rollouts against the full 30-problem set. `num_repeats` @@ -45,13 +47,16 @@ controls rollouts-per-task for pass@k; use 16 for parity-grade evaluation, or dr to 4 for a faster smoke pass: ``` -ng_collect_rollouts \ - +agent_name=hmmt_feb25_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/hmmt_feb25/data/hmmt_feb25_benchmark.jsonl \ - +output_jsonl_fpath=results/hmmt_feb25/rollouts.jsonl \ - +prompt_config=benchmarks/hmmt_feb25/prompts/default.yaml \ - +num_repeats=16 +num_repeats_add_seed=true \ - "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 65536}" +gym eval run --no-serve \ + --agent hmmt_feb25_math_with_judge_simple_agent \ + --input benchmarks/hmmt_feb25/data/hmmt_feb25_benchmark.jsonl \ + --output results/hmmt_feb25/rollouts.jsonl \ + --prompt-config benchmarks/hmmt_feb25/prompts/default.yaml \ + --num-repeats 16 \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 65536 \ + +num_repeats_add_seed=true ``` `num_repeats_add_seed=true` assigns a distinct vLLM `seed` to each rollout via diff --git a/benchmarks/hmmt_nov25/README.md b/benchmarks/hmmt_nov25/README.md index 36cddd227f..f1764247c5 100644 --- a/benchmarks/hmmt_nov25/README.md +++ b/benchmarks/hmmt_nov25/README.md @@ -35,20 +35,22 @@ on the same inputs. ```bash # Prepare benchmark data (downloads from HuggingFace) -ng_prepare_benchmark "+config_paths=[benchmarks/hmmt_nov25/config.yaml]" +gym eval prepare --benchmark hmmt_nov25 # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/hmmt_nov25/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark hmmt_nov25 # Collecting rollouts -ng_collect_rollouts \ - +agent_name=hmmt_nov25_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/hmmt_nov25/data/hmmt_nov25_benchmark.jsonl \ - +output_jsonl_fpath=results/hmmt_nov25_rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/math.yaml \ - +num_repeats=16 \ - +num_repeats_add_seed=true \ - "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 65536}" +gym eval run --no-serve \ + --agent hmmt_nov25_math_with_judge_simple_agent \ + --input benchmarks/hmmt_nov25/data/hmmt_nov25_benchmark.jsonl \ + --output results/hmmt_nov25_rollouts.jsonl \ + --prompt-config benchmarks/prompts/generic/math.yaml \ + --num-repeats 16 \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 65536 \ + +num_repeats_add_seed=true ``` diff --git a/benchmarks/hotpotqa_closedbook/README.md b/benchmarks/hotpotqa_closedbook/README.md index 2f145701a4..5c93080f9d 100644 --- a/benchmarks/hotpotqa_closedbook/README.md +++ b/benchmarks/hotpotqa_closedbook/README.md @@ -19,24 +19,25 @@ validation split — the exact same source used by Skills' (`question`, `expected_answer`, `id`, `type`, `level`). ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/hotpotqa_closedbook/config.yaml]" +gym eval prepare --benchmark hotpotqa_closedbook ``` ## Run servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/hotpotqa_closedbook/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark hotpotqa_closedbook ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=hotpotqa_closedbook_simple_agent \ - +input_jsonl_fpath=benchmarks/hotpotqa_closedbook/data/hotpotqa_closedbook_benchmark.jsonl \ - +output_jsonl_fpath=results/hotpotqa_closedbook_rollouts.jsonl \ - +num_repeats=4 \ +gym eval run --no-serve \ + --agent hotpotqa_closedbook_simple_agent \ + --input benchmarks/hotpotqa_closedbook/data/hotpotqa_closedbook_benchmark.jsonl \ + --output results/hotpotqa_closedbook_rollouts.jsonl \ + --num-repeats 4 \ +num_repeats_add_seed=true ``` diff --git a/benchmarks/human_eval/README.md b/benchmarks/human_eval/README.md index ade0190a0b..36898e2826 100644 --- a/benchmarks/human_eval/README.md +++ b/benchmarks/human_eval/README.md @@ -20,23 +20,23 @@ holds the dataset definition + prompt + prepare script. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/human_eval/config.yaml]" +gym eval prepare --benchmark human_eval # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/human_eval/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark human_eval # Collecting rollouts. The +prompt_config= override is required because # the prepared JSONL stores raw `question` rows (no `responses_create_params.input`); -# ng_collect_rollouts does not pick up the `prompt_config:` field on the dataset -# entry in config.yaml the way ng_run does. -ng_collect_rollouts \ - +agent_name=human_eval_evalplus_simple_agent \ - +input_jsonl_fpath=benchmarks/human_eval/data/human_eval_benchmark.jsonl \ - +prompt_config=benchmarks/prompts/generic/codegen.yaml \ - +output_jsonl_fpath=results/human_eval_rollouts.jsonl \ - +num_repeats=4 +# gym eval run --no-serve does not pick up the `prompt_config:` field on the dataset +# entry in config.yaml the way gym env start does. +gym eval run --no-serve \ + --agent human_eval_evalplus_simple_agent \ + --input benchmarks/human_eval/data/human_eval_benchmark.jsonl \ + --prompt-config benchmarks/prompts/generic/codegen.yaml \ + --output results/human_eval_rollouts.jsonl \ + --num-repeats 4 ``` Start vLLM with `--reasoning-parser ` (e.g. `deepseek_r1` for diff --git a/benchmarks/human_eval_infilling/README.md b/benchmarks/human_eval_infilling/README.md index b5a301e830..be02a182e7 100644 --- a/benchmarks/human_eval_infilling/README.md +++ b/benchmarks/human_eval_infilling/README.md @@ -37,23 +37,23 @@ holds only the dataset definition + prompt + prepare script. ```bash # Prepare benchmark data (downloads all three default splits; # default benchmark variant is random_span) -ng_prepare_benchmark "+config_paths=[benchmarks/human_eval_infilling/config.yaml]" +gym eval prepare --benchmark human_eval_infilling # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/human_eval_infilling/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark human_eval_infilling # Collecting rollouts. The +prompt_config= override is required because # the prepared JSONL stores raw rows (no `responses_create_params.input`); -# ng_collect_rollouts does not pick up the `prompt_config:` field on the -# dataset entry in config.yaml the way ng_run does. -ng_collect_rollouts \ - +agent_name=human_eval_infilling_simple_agent \ - +input_jsonl_fpath=benchmarks/human_eval_infilling/data/random_span.jsonl \ - +prompt_config=benchmarks/human_eval_infilling/prompts/default.yaml \ - +output_jsonl_fpath=results/human_eval_infilling_rollouts.jsonl \ - +num_repeats=1 +# gym eval run --no-serve does not pick up the `prompt_config:` field on the +# dataset entry in config.yaml the way gym env start does. +gym eval run --no-serve \ + --agent human_eval_infilling_simple_agent \ + --input benchmarks/human_eval_infilling/data/random_span.jsonl \ + --prompt-config benchmarks/human_eval_infilling/prompts/default.yaml \ + --output results/human_eval_infilling_rollouts.jsonl \ + --num-repeats 1 ``` ### Other splits @@ -62,13 +62,13 @@ To benchmark a different split, point the resource server at it via the config and feed the matching prepared JSONL: ```bash -ng_collect_rollouts \ - +agent_name=human_eval_infilling_simple_agent \ - '++policy_model.resources_servers.code_fim.split=single_line' \ - +input_jsonl_fpath=benchmarks/human_eval_infilling/data/single_line.jsonl \ - +prompt_config=benchmarks/human_eval_infilling/prompts/default.yaml \ - +output_jsonl_fpath=results/human_eval_infilling_single_line_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent human_eval_infilling_simple_agent \ + --input benchmarks/human_eval_infilling/data/single_line.jsonl \ + --prompt-config benchmarks/human_eval_infilling/prompts/default.yaml \ + --output results/human_eval_infilling_single_line_rollouts.jsonl \ + --num-repeats 1 \ + '++policy_model.resources_servers.code_fim.split=single_line' ``` Start vLLM with `--reasoning-parser ` (e.g. `deepseek_r1` for diff --git a/benchmarks/ifbench/README.md b/benchmarks/ifbench/README.md index 55c5c333c2..2961ecdcd2 100644 --- a/benchmarks/ifbench/README.md +++ b/benchmarks/ifbench/README.md @@ -14,22 +14,23 @@ The `ifbench` resources server clones the AllenAI IFBench repo from GitHub on fi ## Prepare data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/ifbench/config.yaml]" +gym eval prepare --benchmark ifbench ``` ## Run ```bash -ng_e2e_collect_rollouts \ - "+config_paths=[benchmarks/ifbench/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - ++output_jsonl_fpath=results/benchmarks/ifbench.jsonl \ +gym eval run \ + --benchmark ifbench \ + --model-type vllm_model \ + --output results/benchmarks/ifbench.jsonl \ + --split benchmark \ + --model-url <> \ + --model-api-key <> \ + --model <> \ + --resume \ ++overwrite_metrics_conflicts=true \ - ++split=benchmark \ - ++resume_from_cache=true \ - ++reuse_existing_data_preparation=true \ - ++policy_base_url=<> \ - ++policy_api_key=<> \ - ++policy_model_name=<> + ++reuse_existing_data_preparation=true ``` ## No internet access diff --git a/benchmarks/ifeval/README.md b/benchmarks/ifeval/README.md index 891b537178..a7c1626056 100644 --- a/benchmarks/ifeval/README.md +++ b/benchmarks/ifeval/README.md @@ -7,25 +7,25 @@ This benchmark chains to the existing `instruction_following` resources server, ## Prepare data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/ifeval/config.yaml]" +gym eval prepare --benchmark ifeval ``` ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/ifeval/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark ifeval ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=ifeval_instruction_following_simple_agent \ - +input_jsonl_fpath=benchmarks/ifeval/data/ifeval_benchmark.jsonl \ - +output_jsonl_fpath=results/ifeval_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent ifeval_instruction_following_simple_agent \ + --input benchmarks/ifeval/data/ifeval_benchmark.jsonl \ + --output results/ifeval_rollouts.jsonl \ + --num-repeats 4 ``` ## Scoring notes diff --git a/benchmarks/imo_answerbench/README.md b/benchmarks/imo_answerbench/README.md index a2aaf420b4..b1df651d5a 100644 --- a/benchmarks/imo_answerbench/README.md +++ b/benchmarks/imo_answerbench/README.md @@ -7,25 +7,26 @@ Math benchmark from [google-deepmind/superhuman](https://github.com/google-deepm `prepare.py` downloads `answerbench_v2.csv` from the exact pinned superhuman commit that Skills uses, and writes `data/imo_answerbench_benchmark.jsonl` with one row per problem (`question`, `expected_answer`, plus `problem_id` / `category` / `subcategory` / `source`). ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/imo_answerbench/config.yaml]" +gym eval prepare --benchmark imo_answerbench ``` ## Run servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/imo_answerbench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark imo_answerbench ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=imo_answerbench_math_with_autograder_simple_agent \ - +input_jsonl_fpath=benchmarks/imo_answerbench/data/imo_answerbench_benchmark.jsonl \ - +output_jsonl_fpath=results/imo_answerbench_rollouts.jsonl \ - +prompt_config=benchmarks/imo_answerbench/prompts/default.yaml \ - +num_repeats=4 \ +gym eval run --no-serve \ + --agent imo_answerbench_math_with_autograder_simple_agent \ + --input benchmarks/imo_answerbench/data/imo_answerbench_benchmark.jsonl \ + --output results/imo_answerbench_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/imo_answerbench/prompts/default.yaml \ +num_repeats_add_seed=true ``` diff --git a/benchmarks/imo_gradingbench/README.md b/benchmarks/imo_gradingbench/README.md index de46a420b8..3be264572a 100644 --- a/benchmarks/imo_gradingbench/README.md +++ b/benchmarks/imo_gradingbench/README.md @@ -33,20 +33,20 @@ details. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/imo_gradingbench/config.yaml]" +gym eval prepare --benchmark imo_gradingbench # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/imo_gradingbench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark imo_gradingbench # Collecting rollouts (4 rollouts per task) -ng_collect_rollouts \ - +agent_name=imo_gradingbench_imo_gradingbench_simple_agent \ - +input_jsonl_fpath=benchmarks/imo_gradingbench/data/imo_gradingbench_benchmark.jsonl \ - +output_jsonl_fpath=results/imo_gradingbench_rollouts.jsonl \ - +prompt_config=benchmarks/imo_gradingbench/prompts/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent imo_gradingbench_imo_gradingbench_simple_agent \ + --input benchmarks/imo_gradingbench/data/imo_gradingbench_benchmark.jsonl \ + --output results/imo_gradingbench_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/imo_gradingbench/prompts/default.yaml ``` ## Metrics diff --git a/benchmarks/imo_proofbench/README.md b/benchmarks/imo_proofbench/README.md index 3bf4908726..b12bc18d96 100644 --- a/benchmarks/imo_proofbench/README.md +++ b/benchmarks/imo_proofbench/README.md @@ -7,15 +7,15 @@ Math benchmark from [google-deepmind/superhuman](https://github.com/google-deepm `prepare.py` downloads `proofbench.csv` from the exact pinned superhuman commit Skills uses (`c1ee02e03d4cdb2ab21cd01ac927d895f5287fc8`) and writes `data/imo_proofbench_benchmark.jsonl`. Each row carries `problem`, `reference_solution`, `rubric`, plus `problem_id` / `category` / `level` / `expected_answer` / `source`. ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/imo_proofbench/config.yaml]" +gym eval prepare --benchmark imo_proofbench ``` ## Run servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/imo_proofbench/config.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --model-type vllm_model \ + --benchmark imo_proofbench \ +judge_base_url=https://generativelanguage.googleapis.com/v1beta/openai \ "+judge_api_key=$GEMINI_API_KEY" \ +judge_model_name=gemini-2.5-pro @@ -25,16 +25,16 @@ ng_run "+config_paths=[$config_paths]" \ `prepare.py` writes raw problem fields and does **not** bake `responses_create_params.input` into each row. Pass `+prompt_config` so -`ng_collect_rollouts` materialises the chat messages from +`gym eval run --no-serve` materialises the chat messages from `prompts/default.yaml` at rollout time: ```bash -ng_collect_rollouts \ - +agent_name=imo_proofbench_imo_proofbench_judge_simple_agent \ - +prompt_config=benchmarks/imo_proofbench/prompts/default.yaml \ - +input_jsonl_fpath=benchmarks/imo_proofbench/data/imo_proofbench_benchmark.jsonl \ - +output_jsonl_fpath=results/imo_proofbench_rollouts.jsonl \ - +num_repeats=4 \ +gym eval run --no-serve \ + --agent imo_proofbench_imo_proofbench_judge_simple_agent \ + --input benchmarks/imo_proofbench/data/imo_proofbench_benchmark.jsonl \ + --output results/imo_proofbench_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/imo_proofbench/prompts/default.yaml \ +num_repeats_add_seed=true ``` diff --git a/benchmarks/ioi/README.md b/benchmarks/ioi/README.md index 54028ecfef..3fe5680a00 100644 --- a/benchmarks/ioi/README.md +++ b/benchmarks/ioi/README.md @@ -47,15 +47,21 @@ elsewhere. Cluster/SLURM users can co-launch the sandbox via Skills' ## Running ```bash -ng_prepare_data +config_paths=[benchmarks/ioi/config.yaml] \ - +output_dirpath=benchmarks/ioi/data \ - +mode=benchmark_preparation - -ng_run +config_paths=[benchmarks/ioi/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml] - -ng_collect_rollouts +agent_name=ioi_simple_agent \ - +input_jsonl_fpath=benchmarks/ioi/data/ioi24_benchmark.jsonl \ - +output_jsonl_fpath=results/ioi_rollouts.jsonl \ - +num_repeats=50 +num_repeats_add_seed=true \ - "+responses_create_params={max_output_tokens: 131072, temperature: 1.0, top_p: 0.95}" +gym dataset collate --config benchmarks/ioi/config.yaml \ + --output-dir benchmarks/ioi/data \ + --mode benchmark_preparation + +gym env start \ + --benchmark ioi \ + --model-type vllm_model + +gym eval run --no-serve \ + --agent ioi_simple_agent \ + --input benchmarks/ioi/data/ioi24_benchmark.jsonl \ + --output results/ioi_rollouts.jsonl \ + --num-repeats 50 \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 131072 \ + +num_repeats_add_seed=true ``` diff --git a/benchmarks/labbench2_vlm/README.md b/benchmarks/labbench2_vlm/README.md index c506b91c15..ebb17b365b 100644 --- a/benchmarks/labbench2_vlm/README.md +++ b/benchmarks/labbench2_vlm/README.md @@ -50,7 +50,7 @@ https://huggingface.co/settings/tokens, and set `hf_token` in `env.yaml` (see above). ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/labbench2_vlm/config.yaml]" +gym eval prepare --benchmark labbench2_vlm ``` Downloads the subtask splits from HuggingFace and media files (images, @@ -82,10 +82,12 @@ prepared in its validation JSONL but is not included in `example.jsonl`. After changing `example.jsonl`, regenerate its static validation metrics: ```bash -.venv/bin/ng_prepare_data \ - "+config_paths=[resources_servers/labbench2_vlm/configs/labbench2_vlm.yaml,resources_servers/labbench2_vlm/configs/judge_model_openai.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - +mode=example_validation \ - +output_dirpath=/tmp/labbench2_vlm_example_validation \ +.venv/bin/gym dataset collate \ + --resources-server labbench2_vlm \ + --config resources_servers/labbench2_vlm/configs/judge_model_openai.yaml \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --mode example_validation \ + --output-dir /tmp/labbench2_vlm_example_validation \ +overwrite_metrics_conflicts=true ``` @@ -98,18 +100,21 @@ overwrite source data. The full config chain is required because ```bash # Start servers -ng_run "+config_paths=[benchmarks/labbench2_vlm/config.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env start \ + --benchmark labbench2_vlm \ + --model-type openai_model # Collect rollouts -ng_collect_rollouts \ - "+config_paths=[benchmarks/labbench2_vlm/config.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - +agent_name=labbench2_vlm_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/labbench2_vlm/data/labbench2_vlm_benchmark.jsonl \ - +output_jsonl_fpath=results/labbench2_vlm.jsonl +gym eval run --no-serve \ + --benchmark labbench2_vlm \ + --model-type openai_model \ + --agent labbench2_vlm_benchmark_simple_agent \ + --input benchmarks/labbench2_vlm/data/labbench2_vlm_benchmark.jsonl \ + --output results/labbench2_vlm.jsonl ``` `+agent_name` and `+input_jsonl_fpath` are both required — rows in the -prepared JSONL don't carry an `agent_ref`, and `ng_collect_rollouts` doesn't +prepared JSONL don't carry an `agent_ref`, and `gym eval run --no-serve` doesn't read the path from the benchmark config. For **protocolqa2** as **extracted text** with a text-capable policy model, pass @@ -124,20 +129,21 @@ PDFs as pages like other PDF tasks. ### One-shot alternative -`ng_e2e_collect_rollouts` starts the server stack, preprocesses, and -collects rollouts in a single command (don't run `ng_run` separately). +`gym eval run` starts the server stack, preprocesses, and +collects rollouts in a single command (don't run `gym env start` separately). Input path and agent ref are auto-derived from the `type: benchmark` dataset entry in the chained config: ```bash -ng_e2e_collect_rollouts \ - "+config_paths=[benchmarks/labbench2_vlm/config.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - ++split=benchmark \ - ++output_jsonl_fpath=results/labbench2_vlm.jsonl \ - +num_samples_in_parallel=16 +gym eval run \ + --benchmark labbench2_vlm \ + --model-type openai_model \ + --split benchmark \ + --output results/labbench2_vlm.jsonl \ + --concurrency 16 ``` -For a fast smoke test, add `+limit=10 +num_repeats=1`. +For a fast smoke test, add `--limit 10 --num-repeats 1`. `num_repeats` defaults to 3. Bump higher for tighter variance on the judge-based reward. @@ -145,14 +151,14 @@ judge-based reward. ## Throttling Each in-flight sample fans out to one policy call + one judge call, so the -endpoints see roughly `2 × num_samples_in_parallel` concurrent requests. +endpoints see roughly `2 × concurrency` concurrent requests. On a hosted endpoint you'll likely hit rate limits or socket errors (`Hit N global ClientOSError`) well before saturating your machine. -Cap concurrency with `+num_samples_in_parallel=`: +Cap concurrency with `--concurrency `: ```bash -ng_collect_rollouts ... +num_samples_in_parallel=16 +gym eval run --no-serve ... --concurrency 16 ``` Start around 16 and bump up if it holds. diff --git a/benchmarks/librispeech_pc/README.md b/benchmarks/librispeech_pc/README.md index a80b5a4019..676fe50743 100644 --- a/benchmarks/librispeech_pc/README.md +++ b/benchmarks/librispeech_pc/README.md @@ -8,7 +8,7 @@ which provides the WER scoring. ## Splits This benchmark exposes the `test_clean` split (~2.4k utterances). -`ng_prepare_benchmark` enforces one benchmark dataset per agent, so the +`gym eval prepare` enforces one benchmark dataset per agent, so the harder `test_other` split is left for a sibling benchmark dir as a future PR. `prepare.py` accepts `--splits test-other` on the command line and writes a separate `librispeech_pc_test_other.jsonl` if you want to @@ -34,7 +34,7 @@ row. ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/librispeech_pc/config.yaml]" +gym eval prepare --benchmark librispeech_pc ``` Downloads OpenSLR-145 manifests and OpenSLR-12 test-clean audio @@ -43,18 +43,18 @@ and writes the JSONL into `benchmarks/librispeech_pc/data/`. ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/librispeech_pc/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark librispeech_pc ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=librispeech_pc_asr_with_pc_simple_agent \ - +output_jsonl_fpath=results/librispeech_pc_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent librispeech_pc_asr_with_pc_simple_agent \ + --output results/librispeech_pc_rollouts.jsonl \ + --num-repeats 4 ``` ## Verification diff --git a/benchmarks/librispeech_pc/prepare.py b/benchmarks/librispeech_pc/prepare.py index 63dba5b936..3aa9d64699 100644 --- a/benchmarks/librispeech_pc/prepare.py +++ b/benchmarks/librispeech_pc/prepare.py @@ -45,7 +45,7 @@ # test-clean is the standard LibriSpeech-PC eval split; test-other (~2.9k # harder utterances) can be enabled either by adding a sibling benchmark -# dir (the idiomatic shape, since `ng_prepare_benchmark` enforces one +# dir (the idiomatic shape, since `gym eval prepare` enforces one # dataset per agent) or by passing `--splits test-other` and pointing a # separate config at the resulting JSONL. Default below emits only # test-clean. @@ -173,7 +173,7 @@ def prepare(work_dir: Path | None = None, splits: tuple[str, ...] = DEFAULT_SPLI Returns: Path to ``librispeech_pc_test_clean.jsonl`` — the file the benchmark config's ``datasets[0].jsonl_fpath`` references. Returning a single - Path matches the contract ``ng_prepare_benchmark`` enforces. + Path matches the contract ``gym eval prepare`` enforces. """ work_dir = work_dir or DATA_DIR work_dir.mkdir(parents=True, exist_ok=True) diff --git a/benchmarks/livecodebench-x/README.md b/benchmarks/livecodebench-x/README.md index eaefaf4e0e..9002d09f59 100644 --- a/benchmarks/livecodebench-x/README.md +++ b/benchmarks/livecodebench-x/README.md @@ -36,7 +36,7 @@ source the existing monolingual `livecodebench/v5_2408_2502` and ## Data Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/livecodebench-x/config.yaml]" +gym eval prepare --benchmark livecodebench-x ``` That writes `benchmarks/livecodebench-x/data/livecodebench-x_benchmark.jsonl` @@ -47,7 +47,7 @@ have the same characteristic; `code_gen.verify()` already handles it. For a smaller subset (e.g. one language × one version, ~300 rows) suitable for local smoke-testing, invoke the prepare script directly with its argparse -flags — `ng_prepare_benchmark` calls `prepare()` with no kwargs and so cannot +flags — `gym eval prepare` calls `prepare()` with no kwargs and so cannot forward these: ```bash @@ -64,26 +64,32 @@ python benchmarks/livecodebench-x/prepare.py --prompt_language en ## Quickstart ```bash -ng_run "+config_paths=[benchmarks/livecodebench-x/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --benchmark livecodebench-x \ + --model-type vllm_model ``` Then in another shell: ```bash mkdir -p results/livecodebench-x -ng_collect_rollouts \ - "+config_paths=[benchmarks/livecodebench-x/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +agent_name=livecodebench-x_code_gen_simple_agent \ - +input_jsonl_fpath=benchmarks/livecodebench-x/data/livecodebench-x_benchmark.jsonl \ - +output_jsonl_fpath=results/livecodebench-x/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml \ - +num_repeats=4 +num_repeats_add_seed=true \ - "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 16384}" +gym eval run --no-serve \ + --benchmark livecodebench-x \ + --model-type vllm_model \ + --agent livecodebench-x_code_gen_simple_agent \ + --input benchmarks/livecodebench-x/data/livecodebench-x_benchmark.jsonl \ + --output results/livecodebench-x/rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/prompts/generic/default.yaml \ + --temperature 1.0 \ + --top-p 0.95 \ + --max-output-tokens 16384 \ + +num_repeats_add_seed=true ``` -`+config_paths` and `+prompt_config` are required: the prepared JSONL ships +`--config` and `+prompt_config` are required: the prepared JSONL ships raw benchmark rows (no `responses_create_params.input` baked in), and the -agent's dataset-level `prompt_config` is metadata for `ng_run` only — the +agent's dataset-level `prompt_config` is metadata for `gym env start` only — the rollout CLI needs `+prompt_config=...` directly to apply the prompt template before merging `responses_create_params` overrides. `mkdir -p` is needed -because `ng_collect_rollouts` does not create parent directories. +because `gym eval run --no-serve` does not create parent directories. diff --git a/benchmarks/longbench_v2/README.md b/benchmarks/longbench_v2/README.md index 61fbd756ba..36b4233864 100644 --- a/benchmarks/longbench_v2/README.md +++ b/benchmarks/longbench_v2/README.md @@ -42,29 +42,29 @@ python benchmarks/longbench_v2/prepare.py \ ```bash # Prepare benchmark data (default) -ng_prepare_benchmark "+config_paths=[benchmarks/longbench_v2/config.yaml]" +gym eval prepare --benchmark longbench_v2 # Prepare benchmark data (N3 1M variant) -ng_prepare_benchmark "+config_paths=[benchmarks/longbench_v2/config_n3_1m.yaml]" +gym eval prepare --benchmark longbench_v2/config_n3_1m # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/longbench_v2/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark longbench_v2 # Collecting rollouts — default -ng_collect_rollouts \ - +agent_name=longbench_v2_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/longbench_v2/data/longbench_v2_benchmark.jsonl \ - +output_jsonl_fpath=results/longbench_v2_rollouts.jsonl \ - +prompt_config=benchmarks/longbench_v2/prompts/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent longbench_v2_mcqa_simple_agent \ + --input benchmarks/longbench_v2/data/longbench_v2_benchmark.jsonl \ + --output results/longbench_v2_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/longbench_v2/prompts/default.yaml # Collecting rollouts — N3 1M -ng_collect_rollouts \ - +agent_name=longbench_v2_n3_1m_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/longbench_v2/data/longbench_v2_n3_1m_benchmark.jsonl \ - +output_jsonl_fpath=results/longbench_v2_n3_1m_rollouts.jsonl \ - +prompt_config=benchmarks/longbench_v2/prompts/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent longbench_v2_n3_1m_mcqa_simple_agent \ + --input benchmarks/longbench_v2/data/longbench_v2_n3_1m_benchmark.jsonl \ + --output results/longbench_v2_n3_1m_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/longbench_v2/prompts/default.yaml ``` diff --git a/benchmarks/longbench_v2/prepare.py b/benchmarks/longbench_v2/prepare.py index 702b73878f..e341fb344a 100644 --- a/benchmarks/longbench_v2/prepare.py +++ b/benchmarks/longbench_v2/prepare.py @@ -37,7 +37,7 @@ Invocation ---------- -``ng_prepare_benchmark`` calls ``prepare()`` with no arguments, using +``gym eval prepare`` calls ``prepare()`` with no arguments, using the defaults below. To build a custom variant, run this script directly:: diff --git a/benchmarks/longcodebench/README.md b/benchmarks/longcodebench/README.md index 60192ab366..efb0e85ea2 100644 --- a/benchmarks/longcodebench/README.md +++ b/benchmarks/longcodebench/README.md @@ -36,29 +36,29 @@ python benchmarks/longcodebench/prepare.py \ ```bash # Prepare benchmark data (default) -ng_prepare_benchmark "+config_paths=[benchmarks/longcodebench/config.yaml]" +gym eval prepare --benchmark longcodebench # Prepare benchmark data (N3 1M variant) -ng_prepare_benchmark "+config_paths=[benchmarks/longcodebench/config_n3_1m.yaml]" +gym eval prepare --benchmark longcodebench/config_n3_1m # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/longcodebench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark longcodebench # Collecting rollouts — default -ng_collect_rollouts \ - +agent_name=longcodebench_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/longcodebench/data/longcodebench_benchmark.jsonl \ - +output_jsonl_fpath=results/longcodebench_rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent longcodebench_mcqa_simple_agent \ + --input benchmarks/longcodebench/data/longcodebench_benchmark.jsonl \ + --output results/longcodebench_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/prompts/generic/default.yaml # Collecting rollouts — N3 1M -ng_collect_rollouts \ - +agent_name=longcodebench_n3_1m_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/longcodebench/data/longcodebench_n3_1m_benchmark.jsonl \ - +output_jsonl_fpath=results/longcodebench_n3_1m_rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent longcodebench_n3_1m_mcqa_simple_agent \ + --input benchmarks/longcodebench/data/longcodebench_n3_1m_benchmark.jsonl \ + --output results/longcodebench_n3_1m_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/prompts/generic/default.yaml ``` diff --git a/benchmarks/longcodebench/prepare.py b/benchmarks/longcodebench/prepare.py index a650c2a539..dadf4b1bc8 100644 --- a/benchmarks/longcodebench/prepare.py +++ b/benchmarks/longcodebench/prepare.py @@ -35,7 +35,7 @@ Invocation ---------- -``ng_prepare_benchmark`` calls ``prepare()`` with no arguments, using +``gym eval prepare`` calls ``prepare()`` with no arguments, using the defaults below. To build a custom variant, run this script directly:: diff --git a/benchmarks/m_arena_hard/README.md b/benchmarks/m_arena_hard/README.md index 062d8e1688..89835f94b4 100644 --- a/benchmarks/m_arena_hard/README.md +++ b/benchmarks/m_arena_hard/README.md @@ -22,7 +22,7 @@ it back into the prepared JSONL. ## Data Runtime download only — benchmark JSONL is not committed. Run -[`prepare.py`](prepare.py) (or `ng_prepare_benchmark`) to populate +[`prepare.py`](prepare.py) (or `gym eval prepare`) to populate `data/m_arena_hard_benchmark.jsonl`. The prepare script: 1. Calls `datasets.load_dataset("CohereLabs/m-ArenaHard", lang, split="test")` @@ -45,22 +45,22 @@ running `prepare.py`. ```bash # Prepare benchmark data (no baseline -> baseline_answer is empty string) -ng_prepare_benchmark "+config_paths=[benchmarks/m_arena_hard/config.yaml]" +gym eval prepare --benchmark m_arena_hard # Or call prepare.py directly with a baseline JSONL python benchmarks/m_arena_hard/prepare.py --baseline-file path/to/baselines.jsonl # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/m_arena_hard/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark m_arena_hard # Collecting rollouts -ng_collect_rollouts \ - +agent_name=m_arena_hard_arena_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/m_arena_hard/data/m_arena_hard_benchmark.jsonl \ - +output_jsonl_fpath=results/m_arena_hard_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent m_arena_hard_arena_judge_simple_agent \ + --input benchmarks/m_arena_hard/data/m_arena_hard_benchmark.jsonl \ + --output results/m_arena_hard_rollouts.jsonl \ + --num-repeats 4 ``` ## Metrics diff --git a/benchmarks/m_arena_hard/prepare.py b/benchmarks/m_arena_hard/prepare.py index bb92ff41b1..7b849b6f23 100644 --- a/benchmarks/m_arena_hard/prepare.py +++ b/benchmarks/m_arena_hard/prepare.py @@ -32,7 +32,7 @@ When ``--baseline-file`` is omitted, ``baseline_answer`` is set to the empty string so the JSONL is still well-formed for tooling that only needs the question set (e.g. data exploration or Skills-vs-Gym prepare -parity checks). ``ng_prepare_benchmark`` calls ``prepare()`` with no +parity checks). ``gym eval prepare`` calls ``prepare()`` with no arguments and so always takes this no-baseline path; use the script entry-point (``python prepare.py --baseline-file ...``) for full arena_judge runs. @@ -82,7 +82,7 @@ def _format_entry(row: dict, language: str) -> dict: def prepare(languages: list[str] | None = None, baseline_file: str | None = None) -> Path: """Download m-ArenaHard from HF, optionally join baselines, write JSONL. - Called with no arguments by ``ng_prepare_benchmark``. Returns the + Called with no arguments by ``gym eval prepare``. Returns the path to the written JSONL. """ DATA_DIR.mkdir(parents=True, exist_ok=True) diff --git a/benchmarks/m_arena_hard_v2/README.md b/benchmarks/m_arena_hard_v2/README.md index bbf7c38208..03e9805927 100644 --- a/benchmarks/m_arena_hard_v2/README.md +++ b/benchmarks/m_arena_hard_v2/README.md @@ -22,7 +22,7 @@ for the judging protocol and metric details. ## Data Runtime download only — benchmark JSONL is not committed. Run -[`prepare.py`](prepare.py) (or `ng_prepare_benchmark`) to populate +[`prepare.py`](prepare.py) (or `gym eval prepare`) to populate `data/m_arena_hard_v2_benchmark.jsonl`. The prepare script loads the HF dataset across all 23 language configs, iterates each split, and emits one row per `(language, question_id)` with `uid`, `question`, @@ -55,19 +55,19 @@ python benchmarks/m_arena_hard_v2/prepare.py \ ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/m_arena_hard_v2/config.yaml]" +gym eval prepare --benchmark m_arena_hard_v2 # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/m_arena_hard_v2/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark m_arena_hard_v2 # Collecting rollouts -ng_collect_rollouts \ - +agent_name=m_arena_hard_v2_arena_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/m_arena_hard_v2/data/m_arena_hard_v2_benchmark.jsonl \ - +output_jsonl_fpath=results/m_arena_hard_v2_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent m_arena_hard_v2_arena_judge_simple_agent \ + --input benchmarks/m_arena_hard_v2/data/m_arena_hard_v2_benchmark.jsonl \ + --output results/m_arena_hard_v2_rollouts.jsonl \ + --num-repeats 4 ``` ## Metrics diff --git a/benchmarks/math-500/README.md b/benchmarks/math-500/README.md index fdebaa0226..acac10600c 100644 --- a/benchmarks/math-500/README.md +++ b/benchmarks/math-500/README.md @@ -15,17 +15,17 @@ resource server. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/math-500/config.yaml]" +gym eval prepare --benchmark math-500 # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/math-500/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark math-500 # Collecting rollouts -ng_collect_rollouts \ - +agent_name=math_500_math_with_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/math-500/data/math-500_benchmark.jsonl \ - +output_jsonl_fpath=results/math-500/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/math.yaml +gym eval run --no-serve \ + --agent math_500_math_with_judge_simple_agent \ + --input benchmarks/math-500/data/math-500_benchmark.jsonl \ + --output results/math-500/rollouts.jsonl \ + --prompt-config benchmarks/prompts/generic/math.yaml ``` diff --git a/benchmarks/mbpp/README.md b/benchmarks/mbpp/README.md index 45859a3c69..6b511968f1 100644 --- a/benchmarks/mbpp/README.md +++ b/benchmarks/mbpp/README.md @@ -22,23 +22,23 @@ Verification runs in the `evalplus` resource server (shared with ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/mbpp/config.yaml]" +gym eval prepare --benchmark mbpp # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/mbpp/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark mbpp # Collecting rollouts. The +prompt_config= override is required because # the prepared JSONL stores raw `question` rows (no `responses_create_params.input`); -# ng_collect_rollouts does not pick up the `prompt_config:` field on the dataset -# entry in config.yaml the way ng_run does. -ng_collect_rollouts \ - +agent_name=mbpp_evalplus_simple_agent \ - +input_jsonl_fpath=benchmarks/mbpp/data/mbpp_benchmark.jsonl \ - +prompt_config=benchmarks/mbpp/prompts/default.yaml \ - +output_jsonl_fpath=results/mbpp_rollouts.jsonl \ - +num_repeats=4 +# gym eval run --no-serve does not pick up the `prompt_config:` field on the dataset +# entry in config.yaml the way gym env start does. +gym eval run --no-serve \ + --agent mbpp_evalplus_simple_agent \ + --input benchmarks/mbpp/data/mbpp_benchmark.jsonl \ + --output results/mbpp_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/mbpp/prompts/default.yaml ``` Start vLLM with `--reasoning-parser ` (e.g. `deepseek_r1` for diff --git a/benchmarks/minif2f/README.md b/benchmarks/minif2f/README.md index 6569256f2b..f057727457 100644 --- a/benchmarks/minif2f/README.md +++ b/benchmarks/minif2f/README.md @@ -13,7 +13,7 @@ evaluation protocol). ## Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/minif2f/config.yaml]" +gym eval prepare --benchmark minif2f ``` Downloads the source JSONL, splits header from theorem body, strips `sorry` @@ -30,21 +30,22 @@ and set `NEMO_SKILLS_SANDBOX_HOST` / `NEMO_SKILLS_SANDBOX_PORT` before starting the server. ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/minif2f/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark minif2f ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=minif2f_math_formal_lean_simple_agent \ - +input_jsonl_fpath=benchmarks/minif2f/data/minif2f_benchmark.jsonl \ - +output_jsonl_fpath=results/minif2f_rollouts.jsonl \ - +num_repeats=32 \ - +prompt_config=benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent minif2f_math_formal_lean_simple_agent \ + --input benchmarks/minif2f/data/minif2f_benchmark.jsonl \ + --output results/minif2f_rollouts.jsonl \ + --num-repeats 32 \ + --prompt-config benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml \ + --temperature 1.0 \ + --max-output-tokens 16384 ``` Reproduce published miniF2F numbers on a DeepSeek-Prover / Goedel-Prover class diff --git a/benchmarks/mmlu-redux/README.md b/benchmarks/mmlu-redux/README.md index 5c47f85fbc..422d0d3198 100644 --- a/benchmarks/mmlu-redux/README.md +++ b/benchmarks/mmlu-redux/README.md @@ -15,17 +15,17 @@ Migrates NeMo Skills' `mmlu-redux` benchmark to Gym on top of the shared ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/mmlu-redux/config.yaml]" +gym eval prepare --benchmark mmlu-redux # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/mmlu-redux/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark mmlu-redux # Collecting rollouts -ng_collect_rollouts \ - +agent_name=mmlu-redux_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/mmlu-redux/data/mmlu-redux_benchmark.jsonl \ - +output_jsonl_fpath=results/mmlu-redux/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/general-boxed.yaml +gym eval run --no-serve \ + --agent mmlu-redux_mcqa_simple_agent \ + --input benchmarks/mmlu-redux/data/mmlu-redux_benchmark.jsonl \ + --output results/mmlu-redux/rollouts.jsonl \ + --prompt-config benchmarks/prompts/generic/general-boxed.yaml ``` diff --git a/benchmarks/mmlu/README.md b/benchmarks/mmlu/README.md index 6987bf93a3..f4e0b88e8e 100644 --- a/benchmarks/mmlu/README.md +++ b/benchmarks/mmlu/README.md @@ -14,17 +14,17 @@ resource server. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/mmlu/config.yaml]" +gym eval prepare --benchmark mmlu # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/mmlu/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark mmlu # Collecting rollouts -ng_collect_rollouts \ - +agent_name=mmlu_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/mmlu/data/mmlu_benchmark.jsonl \ - +output_jsonl_fpath=results/mmlu/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/eval/aai/mcq-4choices-boxed.yaml +gym eval run --no-serve \ + --agent mmlu_mcqa_simple_agent \ + --input benchmarks/mmlu/data/mmlu_benchmark.jsonl \ + --output results/mmlu/rollouts.jsonl \ + --prompt-config benchmarks/prompts/eval/aai/mcq-4choices-boxed.yaml ``` diff --git a/benchmarks/mmlu_pro/README.md b/benchmarks/mmlu_pro/README.md index a61aa8880c..901174b926 100644 --- a/benchmarks/mmlu_pro/README.md +++ b/benchmarks/mmlu_pro/README.md @@ -13,13 +13,16 @@ This benchmark uses the `mcqa` resource server with the `mcqa_simple_agent`. ```bash # Prepare data -ng_prepare_benchmark "+config_paths=[benchmarks/mmlu_pro/config.yaml]" +gym eval prepare --benchmark mmlu_pro # Start servers -ng_run "+config_paths=[benchmarks/mmlu_pro/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --benchmark mmlu_pro \ + --model-type vllm_model # Collect rollouts -ng_collect_rollouts \ - "+config_paths=[benchmarks/mmlu_pro/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +output_jsonl_fpath=results/mmlu_pro.jsonl +gym eval run --no-serve \ + --benchmark mmlu_pro \ + --model-type vllm_model \ + --output results/mmlu_pro.jsonl ``` diff --git a/benchmarks/mmlu_prox/README.md b/benchmarks/mmlu_prox/README.md index 35676c43a0..07ab5560f2 100644 --- a/benchmarks/mmlu_prox/README.md +++ b/benchmarks/mmlu_prox/README.md @@ -13,13 +13,16 @@ This benchmark uses the `mcqa` resource server with the `mcqa_simple_agent`. ```bash # Prepare data -ng_prepare_benchmark "+config_paths=[benchmarks/mmlu_prox/config.yaml]" +gym eval prepare --benchmark mmlu_prox # Start servers -ng_run "+config_paths=[benchmarks/mmlu_prox/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --benchmark mmlu_prox \ + --model-type vllm_model # Collect rollouts -ng_collect_rollouts \ - "+config_paths=[benchmarks/mmlu_prox/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +output_jsonl_fpath=results/mmlu_prox.jsonl +gym eval run --no-serve \ + --benchmark mmlu_prox \ + --model-type vllm_model \ + --output results/mmlu_prox.jsonl ``` diff --git a/benchmarks/mmmlu/README.md b/benchmarks/mmmlu/README.md index bae74b2969..d62d933c0e 100644 --- a/benchmarks/mmmlu/README.md +++ b/benchmarks/mmmlu/README.md @@ -14,17 +14,17 @@ resource server. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/mmmlu/config.yaml]" +gym eval prepare --benchmark mmmlu # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/mmmlu/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark mmmlu # Collecting rollouts -ng_collect_rollouts \ - +agent_name=mmmlu_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/mmmlu/data/mmmlu_benchmark.jsonl \ - +output_jsonl_fpath=results/mmmlu/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/generic/default.yaml +gym eval run --no-serve \ + --agent mmmlu_mcqa_simple_agent \ + --input benchmarks/mmmlu/data/mmmlu_benchmark.jsonl \ + --output results/mmmlu/rollouts.jsonl \ + --prompt-config benchmarks/prompts/generic/default.yaml ``` diff --git a/benchmarks/mobench/README.md b/benchmarks/mobench/README.md index b32706e6a1..3f43becd65 100644 --- a/benchmarks/mobench/README.md +++ b/benchmarks/mobench/README.md @@ -12,7 +12,7 @@ and `simple_agent` (single-turn, matching NeMo-Skills' evaluation protocol). ## Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/mobench/config.yaml]" +gym eval prepare --benchmark mobench ``` Downloads the source JSONL, splits the prelude from the theorem block via regex, @@ -29,21 +29,22 @@ and set `NEMO_SKILLS_SANDBOX_HOST` / `NEMO_SKILLS_SANDBOX_PORT` before starting the server. ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/mobench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark mobench ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=mobench_math_formal_lean_simple_agent \ - +input_jsonl_fpath=benchmarks/mobench/data/mobench_benchmark.jsonl \ - +output_jsonl_fpath=results/mobench_rollouts.jsonl \ - +num_repeats=32 \ - +prompt_config=benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent mobench_math_formal_lean_simple_agent \ + --input benchmarks/mobench/data/mobench_benchmark.jsonl \ + --output results/mobench_rollouts.jsonl \ + --num-repeats 32 \ + --prompt-config benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml \ + --temperature 1.0 \ + --max-output-tokens 16384 ``` Reproduce published MOBench numbers on a DeepSeek-Prover / Goedel-Prover class diff --git a/benchmarks/mrcr/README.md b/benchmarks/mrcr/README.md index d31ae8ec0c..ebdfabeb38 100644 --- a/benchmarks/mrcr/README.md +++ b/benchmarks/mrcr/README.md @@ -33,44 +33,46 @@ python benchmarks/mrcr/prepare.py \ ```bash # Default (o200k_base, no filter) -ng_prepare_benchmark "+config_paths=[benchmarks/mrcr/config.yaml]" +gym eval prepare --benchmark mrcr # N3 128k variant -ng_prepare_benchmark "+config_paths=[benchmarks/mrcr/config_n3_128k.yaml]" +gym eval prepare --benchmark mrcr/config_n3_128k # N3 1M variant -ng_prepare_benchmark "+config_paths=[benchmarks/mrcr/config_n3_1m.yaml]" +gym eval prepare --benchmark mrcr/config_n3_1m ``` ## Start environment ```bash -ng_run "+config_paths=[benchmarks/mrcr/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --benchmark mrcr \ + --model-type vllm_model ``` ## Collect rollouts ```bash # Default variant -ng_collect_rollouts \ - +agent_name=mrcr_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/mrcr/data/mrcr_benchmark.jsonl \ - +output_jsonl_fpath=results/mrcr_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent mrcr_benchmark_simple_agent \ + --input benchmarks/mrcr/data/mrcr_benchmark.jsonl \ + --output results/mrcr_rollouts.jsonl \ + --num-repeats 4 # N3 128k variant -ng_collect_rollouts \ - +agent_name=mrcr_n3_128k_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/mrcr/data/mrcr_n3_128k_benchmark.jsonl \ - +output_jsonl_fpath=results/mrcr_n3_128k_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent mrcr_n3_128k_benchmark_simple_agent \ + --input benchmarks/mrcr/data/mrcr_n3_128k_benchmark.jsonl \ + --output results/mrcr_n3_128k_rollouts.jsonl \ + --num-repeats 4 # N3 1M variant -ng_collect_rollouts \ - +agent_name=mrcr_n3_1m_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/mrcr/data/mrcr_n3_1m_benchmark.jsonl \ - +output_jsonl_fpath=results/mrcr_n3_1m_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent mrcr_n3_1m_benchmark_simple_agent \ + --input benchmarks/mrcr/data/mrcr_n3_1m_benchmark.jsonl \ + --output results/mrcr_n3_1m_rollouts.jsonl \ + --num-repeats 4 ``` ## Metrics diff --git a/benchmarks/mrcr/prepare.py b/benchmarks/mrcr/prepare.py index 2ef8e767b1..17b6d5d135 100644 --- a/benchmarks/mrcr/prepare.py +++ b/benchmarks/mrcr/prepare.py @@ -31,7 +31,7 @@ Invocation ---------- -``ng_prepare_benchmark`` calls ``prepare()`` with no arguments, using +``gym eval prepare`` calls ``prepare()`` with no arguments, using the defaults below. To build a custom variant, run this script directly:: diff --git a/benchmarks/musan/README.md b/benchmarks/musan/README.md index 4ef8ee8a9d..c317058685 100644 --- a/benchmarks/musan/README.md +++ b/benchmarks/musan/README.md @@ -40,7 +40,7 @@ row. ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/musan/config.yaml]" +gym eval prepare --benchmark musan ``` Downloads the MUSAN OpenSLR archive, extracts it under @@ -53,18 +53,18 @@ points at `/data/musan//audio/musan__NNNNNN.wav` by default ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/musan/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark musan ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=musan_asr_with_pc_simple_agent \ - +output_jsonl_fpath=results/musan_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent musan_asr_with_pc_simple_agent \ + --output results/musan_rollouts.jsonl \ + --num-repeats 4 ``` ## Verification diff --git a/benchmarks/nemotron_3_ultra/README.md b/benchmarks/nemotron_3_ultra/README.md index 7f95ce688c..3aee2957fe 100644 --- a/benchmarks/nemotron_3_ultra/README.md +++ b/benchmarks/nemotron_3_ultra/README.md @@ -36,8 +36,7 @@ Qwen3-235B-A22B-Instruct-2507-FP8: # Prepare benchmark data ```bash -config_paths="benchmarks/nemotron_3_ultra/config_short.yaml" -ng_prepare_benchmark "+config_paths=[$config_paths]" +gym eval prepare --config benchmarks/nemotron_3_ultra/config_short.yaml ``` # Run @@ -65,10 +64,9 @@ This example uses: WANDB_PROJECT=<> EXPERIMENT_NAME=<> -config_paths="benchmarks/nemotron_3_ultra/remote_endpoint.yaml,\ -benchmarks/nemotron_3_ultra/config_short.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ +gym eval run \ + --config benchmarks/nemotron_3_ultra/remote_endpoint.yaml \ + --config benchmarks/nemotron_3_ultra/config_short.yaml \ +wandb_project=$WANDB_PROJECT \ +wandb_name=$EXPERIMENT_NAME \ ++output_jsonl_fpath=results/$EXPERIMENT_NAME.jsonl \ @@ -86,10 +84,9 @@ ng_e2e_collect_rollouts \ WANDB_PROJECT=<> EXPERIMENT_NAME=<> -config_paths="benchmarks/nemotron_3_ultra/local_endpoint.yaml,\ -benchmarks/nemotron_3_ultra/config_short.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ +gym eval run \ + --config benchmarks/nemotron_3_ultra/local_endpoint.yaml \ + --config benchmarks/nemotron_3_ultra/config_short.yaml \ +wandb_project=$WANDB_PROJECT \ +wandb_name=$EXPERIMENT_NAME \ ++output_jsonl_fpath=results/$EXPERIMENT_NAME.jsonl \ @@ -105,10 +102,9 @@ ng_e2e_collect_rollouts \ WANDB_PROJECT=<> EXPERIMENT_NAME=<> -config_paths="benchmarks/nemotron_3_ultra/local_endpoint_no_gpus.yaml,\ -benchmarks/nemotron_3_ultra/config_short_no_gpus.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ +gym eval run \ + --config benchmarks/nemotron_3_ultra/local_endpoint_no_gpus.yaml \ + --config benchmarks/nemotron_3_ultra/config_short_no_gpus.yaml \ +wandb_project=$WANDB_PROJECT \ +wandb_name=$EXPERIMENT_NAME \ ++output_jsonl_fpath=results/$EXPERIMENT_NAME.jsonl \ diff --git a/benchmarks/numb3rs/README.md b/benchmarks/numb3rs/README.md index 206fea0aff..239aa055a2 100644 --- a/benchmarks/numb3rs/README.md +++ b/benchmarks/numb3rs/README.md @@ -54,7 +54,7 @@ row. ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/numb3rs/config.yaml]" +gym eval prepare --benchmark numb3rs ``` Downloads `nvidia/Numb3rs` (split=`test`), iterates the 12 categories, @@ -64,18 +64,18 @@ combined `benchmarks/numb3rs/data/numb3rs_benchmark.jsonl`. ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/numb3rs/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark numb3rs ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=numb3rs_asr_with_pc_simple_agent \ - +output_jsonl_fpath=results/numb3rs_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent numb3rs_asr_with_pc_simple_agent \ + --output results/numb3rs_rollouts.jsonl \ + --num-repeats 4 ``` ## Verification diff --git a/benchmarks/omniscience/README.md b/benchmarks/omniscience/README.md index 725375140e..122b45f2c7 100644 --- a/benchmarks/omniscience/README.md +++ b/benchmarks/omniscience/README.md @@ -10,12 +10,13 @@ Tests factual recall across 6 domains: Humanities, Health, Software Engineering, ```bash # Prepare data -ng_prepare_data +benchmark=omniscience +gym dataset collate +benchmark=omniscience # Run benchmark -ng_collect_rollouts +benchmark=omniscience \ - "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +output_jsonl_fpath=results/omniscience_rollouts.jsonl +gym eval run --no-serve \ + --model-type vllm_model \ + --output results/omniscience_rollouts.jsonl \ + +benchmark=omniscience ``` ## Key Metrics diff --git a/benchmarks/physics/README.md b/benchmarks/physics/README.md index 99057c0e23..6664b9f5cc 100644 --- a/benchmarks/physics/README.md +++ b/benchmarks/physics/README.md @@ -12,7 +12,7 @@ expression, list, set, or option label. ## Prepare data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/physics/config.yaml]" +gym eval prepare --benchmark physics ``` `prepare.py` downloads the dataset, applies the same flatten-and-`\boxed{}` @@ -24,9 +24,9 @@ transformation Skills uses for the multi-part answers, and writes ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/physics/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark physics ``` > Reasoning-model note: start the policy vLLM server with @@ -37,11 +37,11 @@ ng_run "+config_paths=[$config_paths]" ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=physics_physics_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/physics/data/physics_benchmark.jsonl \ - +output_jsonl_fpath=results/physics_rollouts.jsonl \ - +num_repeats=4 \ +gym eval run --no-serve \ + --agent physics_physics_judge_simple_agent \ + --input benchmarks/physics/data/physics_benchmark.jsonl \ + --output results/physics_rollouts.jsonl \ + --num-repeats 4 \ +num_repeats_add_seed=true ``` diff --git a/benchmarks/polymath/README.md b/benchmarks/polymath/README.md index d1606eab3a..e4620107b6 100644 --- a/benchmarks/polymath/README.md +++ b/benchmarks/polymath/README.md @@ -38,7 +38,7 @@ matching Skills. ## Data preparation ``` -ng_prepare_benchmark "+config_paths=[benchmarks/polymath/config.yaml]" +gym eval prepare --benchmark polymath ``` Writes `data/polymath_benchmark.jsonl` with one row per @@ -50,20 +50,20 @@ Start the servers (inherits the `polymath` resources server in symbolic-only mode plus a vLLM model server): ``` -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/polymath/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark polymath ``` Collect rollouts (use `+num_repeats=4` for a quick parity pass, `+num_repeats=16` for parity-grade evaluation): ``` -ng_collect_rollouts \ - +agent_name=polymath_benchmark_simple_agent \ - +input_jsonl_fpath=benchmarks/polymath/data/polymath_benchmark.jsonl \ - +output_jsonl_fpath=results/polymath_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent polymath_benchmark_simple_agent \ + --input benchmarks/polymath/data/polymath_benchmark.jsonl \ + --output results/polymath_rollouts.jsonl \ + --num-repeats 4 ``` Start the model server with `--reasoning-parser ` (e.g. diff --git a/benchmarks/polymath/tests/test_prepare.py b/benchmarks/polymath/tests/test_prepare.py index 1099bd5199..d57635fd46 100644 --- a/benchmarks/polymath/tests/test_prepare.py +++ b/benchmarks/polymath/tests/test_prepare.py @@ -16,7 +16,7 @@ Hits ``format_entry`` with hand-rolled instruction dicts; does NOT network out to HuggingFace or the upstream PolyMath repo (those are -covered by the on-cluster ``ng_prepare_benchmark`` step). +covered by the on-cluster ``gym eval prepare`` step). """ from benchmarks.polymath.prepare import DIFFICULTY_LEVEL_DIC, QUESTION_TEMPLATE, format_entry diff --git a/benchmarks/proof-arena-judge/README.md b/benchmarks/proof-arena-judge/README.md index 19ea6e9456..09b3d132b0 100644 --- a/benchmarks/proof-arena-judge/README.md +++ b/benchmarks/proof-arena-judge/README.md @@ -17,17 +17,17 @@ Adds the `proof-arena-judge` benchmark to Gym. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/proof-arena-judge/config.yaml]" +gym eval prepare --benchmark proof-arena-judge # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/proof-arena-judge/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark proof-arena-judge # Collecting rollouts -ng_collect_rollouts \ - +agent_name=proof_arena_judge_math_proof_judgement_simple_agent \ - +input_jsonl_fpath=benchmarks/proof-arena-judge/data/proof-arena-judge_benchmark.jsonl \ - +output_jsonl_fpath=results/proof-arena-judge/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/judge/math-proof-judge.yaml +gym eval run --no-serve \ + --agent proof_arena_judge_math_proof_judgement_simple_agent \ + --input benchmarks/proof-arena-judge/data/proof-arena-judge_benchmark.jsonl \ + --output results/proof-arena-judge/rollouts.jsonl \ + --prompt-config benchmarks/prompts/judge/math-proof-judge.yaml ``` diff --git a/benchmarks/proof_bench_judge/README.md b/benchmarks/proof_bench_judge/README.md index 0320517a22..3ff92536e5 100644 --- a/benchmarks/proof_bench_judge/README.md +++ b/benchmarks/proof_bench_judge/README.md @@ -34,19 +34,19 @@ details. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/proof_bench_judge/config.yaml]" +gym eval prepare --benchmark proof_bench_judge # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/proof_bench_judge/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark proof_bench_judge # Collecting rollouts (4 rollouts per task) -ng_collect_rollouts \ - +agent_name=proof_bench_judge_math_proof_judgement_simple_agent \ - +input_jsonl_fpath=benchmarks/proof_bench_judge/data/proof_bench_judge_benchmark.jsonl \ - +output_jsonl_fpath=results/proof_bench_judge_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent proof_bench_judge_math_proof_judgement_simple_agent \ + --input benchmarks/proof_bench_judge/data/proof_bench_judge_benchmark.jsonl \ + --output results/proof_bench_judge_rollouts.jsonl \ + --num-repeats 4 ``` ## Metrics diff --git a/benchmarks/proofnet/README.md b/benchmarks/proofnet/README.md index cc80c352eb..9322ab3615 100644 --- a/benchmarks/proofnet/README.md +++ b/benchmarks/proofnet/README.md @@ -12,7 +12,7 @@ and `simple_agent` (single-turn, matching NeMo-Skills' evaluation protocol). ## Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/proofnet/config.yaml]" +gym eval prepare --benchmark proofnet ``` Downloads the source JSONL, filters to `split == "test"`, and writes @@ -28,19 +28,20 @@ and set `NEMO_SKILLS_SANDBOX_HOST` / `NEMO_SKILLS_SANDBOX_PORT` before starting the server. ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/proofnet/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark proofnet ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=proofnet_math_formal_lean_simple_agent \ - +input_jsonl_fpath=benchmarks/proofnet/data/proofnet_benchmark.jsonl \ - +output_jsonl_fpath=results/proofnet_rollouts.jsonl \ - +num_repeats=32 \ - +prompt_config=benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent proofnet_math_formal_lean_simple_agent \ + --input benchmarks/proofnet/data/proofnet_benchmark.jsonl \ + --output results/proofnet_rollouts.jsonl \ + --num-repeats 32 \ + --max-output-tokens 16384 \ + --temperature 1.0 \ + --prompt-config benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml ``` diff --git a/benchmarks/putnam_bench/README.md b/benchmarks/putnam_bench/README.md index 924b067a5d..a286df8830 100644 --- a/benchmarks/putnam_bench/README.md +++ b/benchmarks/putnam_bench/README.md @@ -12,7 +12,7 @@ and `simple_agent` (single-turn, matching NeMo-Skills' evaluation protocol). ## Preparation ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/putnam_bench/config.yaml]" +gym eval prepare --benchmark putnam_bench ``` Clones the upstream repo into a temp dir, runs its `rewrite_solutions.py` subprocess, parses the output `.lean` files, and writes `data/putnam_bench_benchmark.jsonl`. Each row has `name`, `split`, `header`, `informal_prefix`, `formal_statement` (no `goal` field — the `math_formal_lean` server doesn't use it, and the upstream prepare omits it). @@ -28,19 +28,20 @@ and set `NEMO_SKILLS_SANDBOX_HOST` / `NEMO_SKILLS_SANDBOX_PORT` before starting the server. ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/putnam_bench/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark putnam_bench ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=putnam_bench_math_formal_lean_simple_agent \ - +input_jsonl_fpath=benchmarks/putnam_bench/data/putnam_bench_benchmark.jsonl \ - +output_jsonl_fpath=results/putnam_bench_rollouts.jsonl \ - +num_repeats=32 \ - +prompt_config=benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent putnam_bench_math_formal_lean_simple_agent \ + --input benchmarks/putnam_bench/data/putnam_bench_benchmark.jsonl \ + --output results/putnam_bench_rollouts.jsonl \ + --num-repeats 32 \ + --max-output-tokens 16384 \ + --temperature 1.0 \ + --prompt-config benchmarks/prompts/lean4/formal-proof-deepseek-prover-v2.yaml ``` diff --git a/benchmarks/ruler/README.md b/benchmarks/ruler/README.md index 61a9257340..4cac5fc118 100644 --- a/benchmarks/ruler/README.md +++ b/benchmarks/ruler/README.md @@ -5,22 +5,20 @@ Linux: `apt update && apt install -y git-lfs` # Prepare data ```bash -config_paths="benchmarks/ruler/config_nemotron_3_256k.yaml" -ng_prepare_benchmark "+config_paths=[$config_paths]" +gym eval prepare --benchmark ruler/config_nemotron_3_256k ``` # Run ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/ruler/config_nemotron_3_256k.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++output_jsonl_fpath=results/benchmarks/ruler.jsonl \ +gym eval run \ + --model-type vllm_model \ + --benchmark ruler/config_nemotron_3_256k \ + --output results/benchmarks/ruler.jsonl \ + --split benchmark \ + --model-url <> \ + --model-api-key <> \ + --model <> \ + --resume \ ++overwrite_metrics_conflicts=true \ - ++split=benchmark \ - ++resume_from_cache=true \ - ++reuse_existing_data_preparation=true \ - ++policy_base_url=<> \ - ++policy_api_key=<> \ - ++policy_model_name=<> + ++reuse_existing_data_preparation=true ``` diff --git a/benchmarks/ruler/ruler_prepare_script.py b/benchmarks/ruler/ruler_prepare_script.py index 6e9d569f68..ab4deb29e0 100644 --- a/benchmarks/ruler/ruler_prepare_script.py +++ b/benchmarks/ruler/ruler_prepare_script.py @@ -152,7 +152,7 @@ def get_ruler_data( # `nltk.download(...)` branch entirely. # # The lock also coordinates across multiple `get_ruler_data` - # callers (e.g. `ng_prepare_benchmark` pool workers) on the same + # callers (e.g. `gym eval prepare` pool workers) on the same # host — `tempfile.gettempdir()` is `/tmp`, host-shared, and # `flock(2)` is a kernel-level lock, so it serializes correctly # across processes regardless of which Python venv they run in. diff --git a/benchmarks/simpleqa/README.md b/benchmarks/simpleqa/README.md index 8bd24dde26..55230fc3ca 100644 --- a/benchmarks/simpleqa/README.md +++ b/benchmarks/simpleqa/README.md @@ -34,26 +34,26 @@ and derives `f1` + `accuracy_given_attempted` at every aggregation level. ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/simpleqa/config.yaml]" +gym eval prepare --benchmark simpleqa ``` ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/simpleqa/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark simpleqa ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=simpleqa_simpleqa_simple_agent \ - +input_jsonl_fpath=benchmarks/simpleqa/data/simpleqa_benchmark.jsonl \ - +output_jsonl_fpath=results/simpleqa_rollouts.jsonl \ - +prompt_config=benchmarks/simpleqa/prompts/default.yaml \ - +num_repeats=4 +gym eval run --no-serve \ + --agent simpleqa_simpleqa_simple_agent \ + --input benchmarks/simpleqa/data/simpleqa_benchmark.jsonl \ + --output results/simpleqa_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/simpleqa/prompts/default.yaml ``` For reasoning models, start the policy vLLM server with diff --git a/benchmarks/speed-bench/README.md b/benchmarks/speed-bench/README.md index fc7f68db90..596b14eb95 100644 --- a/benchmarks/speed-bench/README.md +++ b/benchmarks/speed-bench/README.md @@ -51,23 +51,23 @@ configurations: # Prepare benchmark data (downloads the upstream HF dataset # nvidia/SPEED-Bench plus the 14 source datasets it interpolates from). # Run on a host that has internet access — see prepare.py for details. -ng_prepare_benchmark "+config_paths=[benchmarks/speed-bench/config_qualitative.yaml]" +gym eval prepare --benchmark speed-bench/config_qualitative # Running servers — uses the local_vllm_model demo config that bakes # ngram speculative decoding into vllm_serve_kwargs.speculative_config. # To use a different target model, swap this for any local_vllm_model # config that includes a `speculative_config:` block. -config_paths="responses_api_models/local_vllm_model/configs/Qwen/Qwen3-30B-A3B-Instruct-2507-ngram-specdec.yaml,\ -benchmarks/speed-bench/config_qualitative.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --model-type local_vllm_model/Qwen/Qwen3-30B-A3B-Instruct-2507-ngram-specdec \ + --benchmark speed-bench/config_qualitative \ +policy_model=Qwen3-30B-A3B-Instruct-2507-ngram-specdec # Collecting rollouts -ng_collect_rollouts \ - +agent_name=speed_bench_qualitative_simple_agent \ - +input_jsonl_fpath=benchmarks/speed-bench/data/speed_bench_qualitative_benchmark.jsonl \ - +output_jsonl_fpath=results/speed_bench_qualitative_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent speed_bench_qualitative_simple_agent \ + --input benchmarks/speed-bench/data/speed_bench_qualitative_benchmark.jsonl \ + --output results/speed_bench_qualitative_rollouts.jsonl \ + --num-repeats 1 ``` If you're using the lighter-weight `vllm_model` config (external vLLM diff --git a/benchmarks/spider2_lite/README.md b/benchmarks/spider2_lite/README.md index cdcd7e03f3..5846b3ccbd 100644 --- a/benchmarks/spider2_lite/README.md +++ b/benchmarks/spider2_lite/README.md @@ -1,22 +1,20 @@ # Prepare data ```bash -config_paths="benchmarks/spider2_lite/config.yaml" -ng_prepare_benchmark "+config_paths=[$config_paths]" +gym eval prepare --benchmark spider2_lite ``` # Run ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/spider2_lite/config.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++output_jsonl_fpath=results/benchmarks/spider2_lite.jsonl \ +gym eval run \ + --model-type vllm_model \ + --benchmark spider2_lite \ + --output results/benchmarks/spider2_lite.jsonl \ + --split benchmark \ + --model-url <> \ + --model-api-key <> \ + --model <> \ + --resume \ ++overwrite_metrics_conflicts=true \ - ++split=benchmark \ ++spider2_lite_benchmark_resources_server.resources_servers.spider2_lite.max_concurrency=8 \ - ++resume_from_cache=true \ - ++reuse_existing_data_preparation=true \ - ++policy_base_url=<> \ - ++policy_api_key=<> \ - ++policy_model_name=<> + ++reuse_existing_data_preparation=true ``` diff --git a/benchmarks/supergpqa/README.md b/benchmarks/supergpqa/README.md index ff3392f728..915cc4cc09 100644 --- a/benchmarks/supergpqa/README.md +++ b/benchmarks/supergpqa/README.md @@ -14,17 +14,17 @@ server. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/supergpqa/config.yaml]" +gym eval prepare --benchmark supergpqa # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/supergpqa/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark supergpqa # Collecting rollouts -ng_collect_rollouts \ - +agent_name=supergpqa_mcqa_simple_agent \ - +input_jsonl_fpath=benchmarks/supergpqa/data/supergpqa_benchmark.jsonl \ - +output_jsonl_fpath=results/supergpqa/rollouts.jsonl \ - +prompt_config=benchmarks/prompts/eval/aai/mcq-10choices.yaml +gym eval run --no-serve \ + --agent supergpqa_mcqa_simple_agent \ + --input benchmarks/supergpqa/data/supergpqa_benchmark.jsonl \ + --output results/supergpqa/rollouts.jsonl \ + --prompt-config benchmarks/prompts/eval/aai/mcq-10choices.yaml ``` diff --git a/benchmarks/ugphysics/README.md b/benchmarks/ugphysics/README.md index a620490c21..7c273675d0 100644 --- a/benchmarks/ugphysics/README.md +++ b/benchmarks/ugphysics/README.md @@ -61,19 +61,21 @@ message. ```bash # Prepare benchmark data -ng_prepare_benchmark "+config_paths=[benchmarks/ugphysics/config.yaml]" +gym eval prepare --benchmark ugphysics # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/ugphysics/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --benchmark ugphysics # Collecting rollouts -ng_collect_rollouts \ - +agent_name=ugphysics_ugphysics_judge_simple_agent \ - +input_jsonl_fpath=benchmarks/ugphysics/data/ugphysics_benchmark.jsonl \ - +output_jsonl_fpath=results/ugphysics_rollouts.jsonl \ - +prompt_config=benchmarks/ugphysics/prompts/default.yaml \ - +num_repeats=4 \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0, top_p: 0.95}" +gym eval run --no-serve \ + --agent ugphysics_ugphysics_judge_simple_agent \ + --input benchmarks/ugphysics/data/ugphysics_benchmark.jsonl \ + --output results/ugphysics_rollouts.jsonl \ + --num-repeats 4 \ + --max-output-tokens 16384 \ + --temperature 1.0 \ + --top-p 0.95 \ + --prompt-config benchmarks/ugphysics/prompts/default.yaml ``` diff --git a/benchmarks/wmt24pp/README.md b/benchmarks/wmt24pp/README.md index 92022c3ecf..6d0f3e622f 100644 --- a/benchmarks/wmt24pp/README.md +++ b/benchmarks/wmt24pp/README.md @@ -14,7 +14,7 @@ details and the Ray GPU-scheduled COMET path. ## Prepare benchmark data ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/wmt24pp/config.yaml]" +gym eval prepare --benchmark wmt24pp ``` In addition to writing `data/wmt24pp_benchmark.jsonl`, the prepare step @@ -32,21 +32,21 @@ runs disable COMET via Hydra override and rely on corpus-BLEU only; xCOMET scoring still works end-to-end on the cluster path: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/wmt24pp/config.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --model-type vllm_model \ + --benchmark wmt24pp \ "++wmt24pp_wmt_translation_resources_server.resources_servers.wmt_translation.compute_comet=false" ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=wmt24pp_wmt_translation_simple_agent \ - +prompt_config=benchmarks/wmt24pp/prompts/default.yaml \ - +input_jsonl_fpath=benchmarks/wmt24pp/data/wmt24pp_benchmark.jsonl \ - +output_jsonl_fpath=results/wmt24pp_rollouts.jsonl \ - +num_repeats=4 +gym eval run --no-serve \ + --agent wmt24pp_wmt_translation_simple_agent \ + --input benchmarks/wmt24pp/data/wmt24pp_benchmark.jsonl \ + --output results/wmt24pp_rollouts.jsonl \ + --num-repeats 4 \ + --prompt-config benchmarks/wmt24pp/prompts/default.yaml ``` ## End-to-end reproduction on a SLURM cluster (via NeMo-Skills) @@ -98,7 +98,7 @@ The two container fields that aren't trivial: #### Prepare benchmark data on the cluster -The local `ng_prepare_benchmark` from [above](#prepare-benchmark-data) +The local `gym eval prepare` from [above](#prepare-benchmark-data) writes the JSONL to your dev workstation. For a SLURM run, the JSONL plus the `Unbabel/XCOMET-XXL` cache need to live on the cluster's filesystem. Dispatch the prepare via `ns run_cmd` with the `nemo-gym` @@ -110,7 +110,7 @@ ns run_cmd \ --cluster \ --container nemo-gym \ --expname wmt24pp_prepare \ - --command 'ng_prepare_benchmark "+config_paths=[benchmarks/wmt24pp/config.yaml]"' + --command 'gym eval prepare --benchmark wmt24pp' ``` This populates `benchmarks/wmt24pp/data/wmt24pp_benchmark.jsonl` and diff --git a/benchmarks/xstest/README.md b/benchmarks/xstest/README.md index 0e8372cc26..a1bd1fdcfb 100644 --- a/benchmarks/xstest/README.md +++ b/benchmarks/xstest/README.md @@ -3,22 +3,20 @@ Running this benchmark requires 1 GPU for 7B https://huggingface.co/allenai/wild # Prepare data ```bash -config_paths="benchmarks/xstest/config.yaml" -ng_prepare_benchmark "+config_paths=[$config_paths]" +gym eval prepare --benchmark xstest ``` # Run ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -benchmarks/xstest/config.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++output_jsonl_fpath=results/benchmarks/xstest.jsonl \ +gym eval run \ + --model-type vllm_model \ + --benchmark xstest \ + --output results/benchmarks/xstest.jsonl \ + --split benchmark \ + --model-url <> \ + --model-api-key <> \ + --model <> \ + --resume \ ++overwrite_metrics_conflicts=true \ - ++split=benchmark \ - ++resume_from_cache=true \ - ++reuse_existing_data_preparation=true \ - ++policy_base_url=<> \ - ++policy_api_key=<> \ - ++policy_model_name=<> + ++reuse_existing_data_preparation=true ``` diff --git a/environments/abstention/README.md b/environments/abstention/README.md index cd68ed8973..00418656b3 100644 --- a/environments/abstention/README.md +++ b/environments/abstention/README.md @@ -15,20 +15,20 @@ The dataset used is [HotPotQA](https://hotpotqa.github.io/) (fullwiki split). ## Running servers ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -environments/abstention/config.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --environment abstention \ + --model-type openai_model \ +abstention.resources_servers.abstention.judge_model_server.name=policy_model ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=abstention_simple_agent \ - +input_jsonl_fpath=environments/abstention/data/example.jsonl \ - +output_jsonl_fpath=results/abstention_verify_responses.jsonl \ - +limit=3 +gym eval run --no-serve \ + --agent abstention_simple_agent \ + --input environments/abstention/data/example.jsonl \ + --output results/abstention_verify_responses.jsonl \ + --limit 3 ``` ## Preprocessing HotPotQA data diff --git a/environments/arc_agi/README.md b/environments/arc_agi/README.md index 86eac0ef72..84cb034fc1 100644 --- a/environments/arc_agi/README.md +++ b/environments/arc_agi/README.md @@ -51,7 +51,7 @@ uv sync ### Start ARC-AGI environment (we can reuse the same one for ARC-AGI-1 and 2): ```bash -ng_run "+config_paths=[environments/arc_agi/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start --environment arc_agi --model-type vllm_model ``` @@ -59,12 +59,12 @@ ng_run "+config_paths=[environments/arc_agi/config.yaml,responses_api_models/vll ARC-AGI-1 example rollouts: ```bash -ng_collect_rollouts +agent_name=arc_agi_simple_agent +input_jsonl_fpath=environments/arc_agi/data/example_1.jsonl +output_jsonl_fpath=environments/arc_agi/data/example_1_rollouts.jsonl +gym eval run --no-serve --agent arc_agi_simple_agent --input environments/arc_agi/data/example_1.jsonl --output environments/arc_agi/data/example_1_rollouts.jsonl ``` ARC-AGI-2 example rollouts: ```bash -ng_collect_rollouts +agent_name=arc_agi_simple_agent +input_jsonl_fpath=environments/arc_agi/data/example_2.jsonl +output_jsonl_fpath=environments/arc_agi/data/example_2_rollouts.jsonl +gym eval run --no-serve --agent arc_agi_simple_agent --input environments/arc_agi/data/example_2.jsonl --output environments/arc_agi/data/example_2_rollouts.jsonl ``` For training, see the [docs](https://docs.nvidia.com/nemo/gym/latest/training-tutorials/nemo-rl-grpo/index.html). \ No newline at end of file diff --git a/environments/blackjack/README.md b/environments/blackjack/README.md index adfecbb546..bcbedeab64 100644 --- a/environments/blackjack/README.md +++ b/environments/blackjack/README.md @@ -9,7 +9,7 @@ Example data provided in `data/example.jsonl` (system prompt only, no verifier_m ## Run ```bash -ng_run "+config_paths=[environments/blackjack/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start --environment blackjack --model-type vllm_model ``` ## Data @@ -19,10 +19,10 @@ Each game is generated on the fly during `reset()`, so every row in `example.jso ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=blackjack_gymnasium_agent \ - +input_jsonl_fpath=environments/blackjack/data/example.jsonl \ - +output_jsonl_fpath=results/blackjack_rollouts.jsonl +gym eval run --no-serve \ + --agent blackjack_gymnasium_agent \ + --input environments/blackjack/data/example.jsonl \ + --output results/blackjack_rollouts.jsonl ``` diff --git a/environments/calendar/README.md b/environments/calendar/README.md index de05ef056a..0431490fb1 100644 --- a/environments/calendar/README.md +++ b/environments/calendar/README.md @@ -16,9 +16,7 @@ Create an `env.yaml` file in the Gym root directory to specifying `policy_base_u The following is an example command for running this resources server along with an OpenAI model: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -environments/calendar/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start --environment calendar --model-type openai_model ``` ## Collecting rollouts @@ -26,11 +24,11 @@ ng_run "+config_paths=[$config_paths]" Rollouts can be collected using the example dataset as follows: ```bash -ng_collect_rollouts \ - +agent_name=calendar_simple_agent \ - +input_jsonl_fpath=environments/calendar/data/example.jsonl \ - +output_jsonl_fpath=results/example_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent calendar_simple_agent \ + --input environments/calendar/data/example.jsonl \ + --output results/example_rollouts.jsonl \ + --limit 5 ``` The input JSONL file should contain entries with: diff --git a/environments/calendar_v2/README.md b/environments/calendar_v2/README.md index 0e9138f9f8..2f2586e06f 100644 --- a/environments/calendar_v2/README.md +++ b/environments/calendar_v2/README.md @@ -16,9 +16,7 @@ The conversations in the dataset are generated using personas from the [nvidia/N The following is an example command for running this resources server along with an OpenAI model: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -environments/calendar_v2/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start --environment calendar_v2 --model-type openai_model ``` ## Collecting rollouts @@ -26,11 +24,11 @@ Create an `env.yaml` file in the Gym root directory to specifying `policy_base_u Rollouts can be collected using the example dataset as follows: ```bash -ng_collect_rollouts \ - +agent_name=calendar_simple_agent \ - +input_jsonl_fpath=data/example.jsonl \ - +output_jsonl_fpath=environments/calendar_v2/data/example.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent calendar_simple_agent \ + --input data/example.jsonl \ + --output environments/calendar_v2/data/example.jsonl \ + --limit 5 ``` The input JSONL file should contain entries with: diff --git a/environments/circle_click/README.md b/environments/circle_click/README.md index f83297a676..eeb7a3c052 100644 --- a/environments/circle_click/README.md +++ b/environments/circle_click/README.md @@ -12,8 +12,8 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & -ng_run "+config_paths=[environments/circle_click/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" & -ng_collect_rollouts +agent_name=circle_click_simple_agent +input_jsonl_fpath=environments/circle_click/data/example.jsonl +output_jsonl_fpath=environments/circle_click/data/example_rollouts.jsonl +limit=1 +gym env start --environment circle_click --model-type vllm_model & +gym eval run --no-serve --agent circle_click_simple_agent --input environments/circle_click/data/example.jsonl --output environments/circle_click/data/example_rollouts.jsonl --limit 1 ``` # Generating Data diff --git a/environments/circle_count/README.md b/environments/circle_count/README.md index bc7de17d45..691e45fe54 100644 --- a/environments/circle_count/README.md +++ b/environments/circle_count/README.md @@ -19,8 +19,8 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & -ng_run "+config_paths=[environments/circle_count/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" & -ng_collect_rollouts +agent_name=circle_count_simple_agent +input_jsonl_fpath=environments/circle_count/data/example.jsonl +output_jsonl_fpath=environments/circle_count/data/example_rollouts.jsonl +limit=1 +gym env start --environment circle_count --model-type vllm_model & +gym eval run --no-serve --agent circle_count_simple_agent --input environments/circle_count/data/example.jsonl --output environments/circle_count/data/example_rollouts.jsonl --limit 1 ``` # Generating Data diff --git a/environments/code_gen/README.md b/environments/code_gen/README.md index 1b10d07744..8a4b86fd4b 100644 --- a/environments/code_gen/README.md +++ b/environments/code_gen/README.md @@ -21,18 +21,15 @@ The dataset is in preparation and the example data can be found in `data/example We use the LiveCodeBench execution code. ### Example of rollouts and usage -Create an `env.yaml` file in the Gym root directory to specify the model endpoint and credentials. See [documentation](https://docs.nvidia.com/nemo/gym/reference/configuration#local-configuration-envyaml) for details. +Create an `env.yaml` file in the Gym root directory to specify the model endpoint and credentials. See [documentation](https://docs.nvidia.com/nemo/gym/reference/configuration#local-configuration-envyaml) for details. ```bash # Running the server -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -environments/code_gen/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start --environment code_gen --model-type openai_model # Collect rollouts from example problems -ng_collect_rollouts +agent_name=code_gen_simple_agent \ - +input_jsonl_fpath=environments/code_gen/data/example.jsonl \ - +output_jsonl_fpath=environments/code_gen/data/example_rollouts.jsonl \ - +limit=null +gym eval run --no-serve --agent code_gen_simple_agent \ + --input environments/code_gen/data/example.jsonl \ + --output environments/code_gen/data/example_rollouts.jsonl ``` ## Licensing information diff --git a/environments/ether0/README.md b/environments/ether0/README.md index 1a77748666..e7943d7b10 100644 --- a/environments/ether0/README.md +++ b/environments/ether0/README.md @@ -27,13 +27,13 @@ Start servers and collect rollouts ```bash # start vllm and nemo gym servers vllm serve futurehouse/ether0 & -ng_run "+config_paths=[environments/ether0/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" & +gym env start --environment ether0 --model-type vllm_model & # wait for above to be ready -ng_collect_rollouts \ - +agent_name=ether0_simple_agent \ - +input_jsonl_fpath=environments/ether0/data/example.jsonl \ - +output_jsonl_fpath=environments/ether0/data/ether0_rollouts.jsonl +gym eval run --no-serve \ + --agent ether0_simple_agent \ + --input environments/ether0/data/example.jsonl \ + --output environments/ether0/data/ether0_rollouts.jsonl tail -n 1 environments/ether0/data/ether0_rollouts.jsonl | jq | less ``` diff --git a/environments/instruction_following/README.md b/environments/instruction_following/README.md index 965ecbb67f..575073b910 100644 --- a/environments/instruction_following/README.md +++ b/environments/instruction_following/README.md @@ -47,14 +47,14 @@ The dataset can be found at https://huggingface.co/datasets/nvidia/Nemotron-RL-i ### Usage Create an `env.yaml` file in the Gym root directory to specifying `policy_base_url`, `policy_model_name`, and `policy_api_key`. See [documentation](https://docs.nvidia.com/nemo/gym/reference/configuration#local-configuration-envyaml) for details. ```bash -ng_run "+config_paths=[environments/instruction_following/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start --environment instruction_following --model-type vllm_model python environments/instruction_following/prepare.py -ng_collect_rollouts \ - +agent_name=instruction_following_simple_agent \ - +input_jsonl_fpath=environments/instruction_following/data/example.jsonl \ - +output_jsonl_fpath=results/instruction_following_rollouts.jsonl +limit=5 +gym eval run --no-serve \ + --agent instruction_following_simple_agent \ + --input environments/instruction_following/data/example.jsonl \ + --output results/instruction_following_rollouts.jsonl --limit 5 ``` ### Rollout example diff --git a/environments/reasoning_gym/README.md b/environments/reasoning_gym/README.md index c3f866f14f..dac145fc39 100644 --- a/environments/reasoning_gym/README.md +++ b/environments/reasoning_gym/README.md @@ -87,15 +87,15 @@ policy_model_name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 ## Launch nemo gym servers ```bash -ng_run "+config_paths=[environments/reasoning_gym/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start --environment reasoning_gym --model-type vllm_model ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=reasoning_gym_simple_agent \ - +input_jsonl_fpath=environments/reasoning_gym/data/example.jsonl \ - +output_jsonl_fpath=results/reasoning_gym_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent reasoning_gym_simple_agent \ + --input environments/reasoning_gym/data/example.jsonl \ + --output results/reasoning_gym_rollouts.jsonl \ + --limit 5 ``` diff --git a/environments/workplace_assistant/README.md b/environments/workplace_assistant/README.md index a714aa5f4b..2be692c8bf 100644 --- a/environments/workplace_assistant/README.md +++ b/environments/workplace_assistant/README.md @@ -11,17 +11,18 @@ Commands - Spin up server: ``` -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -environments/workplace_assistant/config.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --environment workplace_assistant ``` Collect trajectories: ``` -ng_collect_rollouts +agent_name=workplace_assistant_simple_agent \ - +input_jsonl_fpath=environments/workplace_assistant/data/example.jsonl \ - +output_jsonl_fpath=results/workplace_assistant_trajectory_collection.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent workplace_assistant_simple_agent \ + --input environments/workplace_assistant/data/example.jsonl \ + --output results/workplace_assistant_trajectory_collection.jsonl \ + --limit 1 ``` ## Generating Additional Training Data diff --git a/fern/versions/latest/pages/contribute/development-setup.mdx b/fern/versions/latest/pages/contribute/development-setup.mdx index a286ec53fe..925685db24 100644 --- a/fern/versions/latest/pages/contribute/development-setup.mdx +++ b/fern/versions/latest/pages/contribute/development-setup.mdx @@ -25,9 +25,9 @@ pre-commit install **Run NeMo Gym Tests**: ```bash -ng_dev_test # Run all NeMo Gym core tests -ng_test_all # Run all server tests -ng_test +entrypoint=responses_api_agents/simple_agent # Test single server +gym dev test # Run all NeMo Gym core tests +gym env test # Run all server tests +gym env test --resources-server example_single_tool_call # Test a single resources server ``` **View Test Coverage**: @@ -37,7 +37,7 @@ coverage html # Generate HTML coverage report **Configuration Debugging**: ```bash -ng_dump_config # Dump config as NeMo Gym sees it +gym env resolve # Dump config as NeMo Gym sees it ``` --- @@ -55,7 +55,7 @@ All contributions must pass these automated checks: **Test Requirements**: - At least one test per server you contribute -- Tests must run using `ng_test +entrypoint=your_server_path` +- Tests must run using `gym env test --resources-server your_server` - Use pytest for async testing patterns ### Build Docs CI Failures @@ -246,7 +246,7 @@ For a small, self-contained change, open a single PR: 1. **Create a feature branch**: `git checkout -b /` 2. **Make changes** with tests -3. **Run local checks**: `ng_dev_test && pre-commit run --all-files` +3. **Run local checks**: `gym dev test && pre-commit run --all-files` 4. **Commit with signoff**: `git commit -s -S -m "Your message"` 5. **Push and open a PR**: ensure all CI checks pass 6. **Address review feedback** and iterate diff --git a/fern/versions/latest/pages/contribute/environments/new-environment.mdx b/fern/versions/latest/pages/contribute/environments/new-environment.mdx index e2f4af4db9..a57d1f71bc 100644 --- a/fern/versions/latest/pages/contribute/environments/new-environment.mdx +++ b/fern/versions/latest/pages/contribute/environments/new-environment.mdx @@ -63,7 +63,7 @@ Prepare the dataset for your environment: Build your resources server: -- Run `ng_init_resources_server +entrypoint=resources_servers/my_server` to scaffold the new resources server +- Run `gym env init --resources-server my_server` to scaffold the new resources server - Follow the [Single Step Environment](/environment-tutorials/single-step-environment) guide to implement your specific logic - Implement verification logic for your tasks by defining the `verify()` function - Set the `domain` field in your resources server configuration (see `Domain`). @@ -80,7 +80,7 @@ Write and run tests for your resources server: Verify basic functionality and generate example rollouts: -- Document the command used to start your server, for example, `ng_run +entrypoint=resources_servers/my_server` +- Document the command used to start your server, for example, `gym env start --resources-server my_server` - Generate rollouts and save 5 example outputs to `data/example_rollouts.jsonl` to demonstrate correct reward signals ### 5. Reward Profiling diff --git a/fern/versions/latest/pages/data/download-huggingface.mdx b/fern/versions/latest/pages/data/download-huggingface.mdx index b49cb97b51..bc6f3337e4 100644 --- a/fern/versions/latest/pages/data/download-huggingface.mdx +++ b/fern/versions/latest/pages/data/download-huggingface.mdx @@ -14,10 +14,10 @@ Download JSONL datasets from Hugging Face Hub for NeMo Gym training. ## Quick Start ```bash -ng_download_dataset_from_hf \ - +repo_id=nvidia/Nemotron-RL-math-OpenMathReasoning \ - +split=train \ - +output_fpath=./data/train.jsonl +gym dataset download \ + --repo-id nvidia/Nemotron-RL-math-OpenMathReasoning \ + --split train \ + --output ./data/train.jsonl ``` ```text @@ -55,9 +55,9 @@ Downloads using the `datasets` library and converts to JSONL. **All splits**: ```bash -ng_download_dataset_from_hf \ - +repo_id=nvidia/Nemotron-RL-knowledge-mcqa \ - +output_dirpath=./data/ +gym dataset download \ + --repo-id nvidia/Nemotron-RL-knowledge-mcqa \ + --output-dir ./data/ ``` ```text @@ -68,10 +68,10 @@ ng_download_dataset_from_hf \ **Single split**: ```bash -ng_download_dataset_from_hf \ - +repo_id=SWE-Gym/SWE-Gym \ - +split=train \ - +output_fpath=./data/train.jsonl +gym dataset download \ + --repo-id SWE-Gym/SWE-Gym \ + --split train \ + --output ./data/train.jsonl ``` @@ -82,10 +82,10 @@ Downloads a specific file directly without conversion. **Use when**: Repository contains pre-formatted JSONL files. ```bash -ng_download_dataset_from_hf \ - +repo_id=nvidia/nemotron-RL-coding-competitive_coding \ - +artifact_fpath=opencodereasoning_filtered_25k_train.jsonl \ - +output_fpath=./data/train.jsonl +gym dataset download \ + --repo-id nvidia/nemotron-RL-coding-competitive_coding \ + --artifact opencodereasoning_filtered_25k_train.jsonl \ + --output ./data/train.jsonl ``` ```text @@ -208,9 +208,9 @@ Avoid passing tokens on the command line—they appear in shell history. ```bash export HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxxxxxxx -ng_download_dataset_from_hf \ - +repo_id=my-org/private-dataset \ - +output_dirpath=./data/ +gym dataset download \ + --repo-id my-org/private-dataset \ + --output-dir ./data/ ``` Get your token at [huggingface.co/settings/tokens](https://huggingface.co/settings/tokens). Use a **read-only** token. @@ -220,10 +220,10 @@ Get your token at [huggingface.co/settings/tokens](https://huggingface.co/settin Not recommended for shared systems: ```bash -ng_download_dataset_from_hf \ - +repo_id=my-org/private-dataset \ - +hf_token=hf_xxxxxxxxxxxxxxxxxxxxxxxxx \ - +output_dirpath=./data/ +gym dataset download \ + --repo-id my-org/private-dataset \ + --output-dir ./data/ \ + +hf_token=hf_xxxxxxxxxxxxxxxxxxxxxxxxx ``` @@ -245,11 +245,11 @@ datasets: Run with download enabled: ```bash -config_paths="resources_servers/code_gen/configs/code_gen.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=./data/prepared \ - +mode=train_preparation \ - +should_download=true \ +gym dataset collate \ + --resources-server code_gen \ + --output-dir ./data/prepared \ + --mode train_preparation \ + --download \ +data_source=huggingface ``` @@ -286,7 +286,7 @@ rm -rf ~/.cache/huggingface/hub/datasets---- -Preprocess raw data, run `ng_prepare_data`, and add `agent_ref` routing. +Preprocess raw data, run `gym dataset collate`, and add `agent_ref` routing. diff --git a/fern/versions/latest/pages/data/index.mdx b/fern/versions/latest/pages/data/index.mdx index 0fe7f47b0e..0bd227a052 100644 --- a/fern/versions/latest/pages/data/index.mdx +++ b/fern/versions/latest/pages/data/index.mdx @@ -34,7 +34,7 @@ Additional fields like `expected_answer` vary by resources server—the componen | Field | Added By | Description | |-------|----------|-------------| | `responses_create_params` | User | Input to the model during training. Contains `input` (messages) and optional `tools`, `temperature`, etc. | -| `agent_ref` | `ng_prepare_data` | Routes each row to its agent server. Auto-generated during data preparation. | +| `agent_ref` | `gym dataset collate` | Routes each row to its agent server. Auto-generated during data preparation. | ### Optional Fields @@ -60,7 +60,7 @@ The `agent_ref` field maps each row to a specific agent server, which in turn kn } ``` -**You don't create `agent_ref` manually.** The `ng_prepare_data` tool adds it automatically based on your config file. The tool matches the agent type (`responses_api_agents`) with the agent name from the config. +**You don't create `agent_ref` manually.** The `gym dataset collate` tool adds it automatically based on your config file. The tool matches the agent type (`responses_api_agents`) with the agent name from the config. ### Example Data @@ -75,12 +75,11 @@ The `agent_ref` field maps each row to a specific agent server, which in turn kn Run this command from the repository root: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/example_multi_step/configs/example_multi_step.yaml" - -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/test \ - +mode=example_validation +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resources-server example_multi_step \ + --output-dir data/test \ + --mode example_validation ``` **Success**: `Finished!` message and `data/test/example_metrics.json` created. @@ -126,7 +125,7 @@ datasets: ```mermaid flowchart LR A[Create JSONL] --> B[Add to config] - B --> C[Run ng_prepare_data] + B --> C[Run gym dataset collate] C -->|Pass| D[Train with NeMo RL] C -->|Fail| E[Fix and retry] ``` @@ -141,13 +140,12 @@ flowchart LR To prepare training data with auto-download: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/workplace_assistant/configs/workplace_assistant.yaml" - -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/workplace_assistant \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resources-server workplace_assistant \ + --output-dir data/workplace_assistant \ + --mode train_preparation \ + --download ``` @@ -162,7 +160,7 @@ HuggingFace downloads require authentication. Set `hf_token` in `env.yaml` or ex | `JSON parse error at line N` | Invalid JSON | Check quotes, commas, brackets at line N | | `ValidationError: responses_create_params` | Missing field | Add `responses_create_params.input` | | `A license is required` | Missing license | Add `license` to dataset config | -| `Missing local datasets` | File not found | Check path or add `+should_download=true` | +| `Missing local datasets` | File not found | Check path or add `--download` | ## Guides @@ -192,8 +190,8 @@ YAML-based prompt templates applied at rollout time. | Command | Description | |---------|-------------| -| `ng_prepare_data` | Validate and generate metrics | -| `ng_download_dataset_from_hf` | Download from HuggingFace | +| `gym dataset collate` | Validate and generate metrics | +| `gym dataset download` | Download from HuggingFace | See [CLI Commands](/reference/cli-commands) for details. diff --git a/fern/versions/latest/pages/data/prepare-validate.mdx b/fern/versions/latest/pages/data/prepare-validate.mdx index 3c024e6e95..e32bbad1f7 100644 --- a/fern/versions/latest/pages/data/prepare-validate.mdx +++ b/fern/versions/latest/pages/data/prepare-validate.mdx @@ -3,7 +3,7 @@ title: "Prepare and Validate" description: "" position: 2 --- -Format and validate JSONL datasets for NeMo Gym training using `ng_prepare_data`. +Format and validate JSONL datasets for NeMo Gym training using `gym dataset collate`. @@ -13,7 +13,7 @@ Format and validate JSONL datasets for NeMo Gym training using `ng_prepare_data` **In this guide, you will**: -1. Validate datasets with `ng_prepare_data` +1. Validate datasets with `gym dataset collate` 2. Generate training and validation splits 3. Understand the JSONL data format @@ -30,12 +30,11 @@ Format and validate JSONL datasets for NeMo Gym training using `ng_prepare_data` From the repository root: ```bash -config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_prepare_data \ - "+config_paths=[$config_paths]" \ - +output_dirpath=data/test \ - +mode=example_validation +gym dataset collate \ + --resources-server example_multi_step \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/test \ + --mode example_validation ``` Success output: @@ -106,7 +105,7 @@ Check `resources_servers//README.md` for required fields specific to each ## Preprocess Raw Datasets -If your dataset doesn't have `responses_create_params`, you need to preprocess it before using `ng_prepare_data`. +If your dataset doesn't have `responses_create_params`, you need to preprocess it before using `gym dataset collate`. **When to preprocess**: - Downloaded datasets without NeMo Gym format @@ -178,7 +177,7 @@ wc -l train.jsonl validation.jsonl ### Create Config for Custom Data -After preprocessing, create a config file to point `ng_prepare_data` at your local files. +After preprocessing, create a config file to point `gym dataset collate` at your local files. @@ -216,8 +215,11 @@ custom_simple_agent: Run data preparation: ```bash -config_paths="custom_data.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" +mode=train_preparation +output_dirpath=data +gym dataset collate \ + --config custom_data.yaml \ + --config responses_api_models/vllm_model/configs/vllm_model.yaml \ + --mode train_preparation \ + --output-dir data ``` This validates your data and adds the `agent_ref` field to each row, routing samples to your resources server. @@ -234,28 +236,32 @@ This validates your data and adds the `agent_ref` field to each row, routing sam ### Example Validation ```bash -ng_prepare_data "+config_paths=[resources_servers/example_multi_step/configs/example_multi_step.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +output_dirpath=data/example_multi_step \ - +mode=example_validation +gym dataset collate \ + --resources-server example_multi_step \ + --config responses_api_models/vllm_model/configs/vllm_model.yaml \ + --output-dir data/example_multi_step \ + --mode example_validation ``` ### Training Preparation ```bash -ng_prepare_data "+config_paths=[resources_servers/workplace_assistant/configs/workplace_assistant.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - +output_dirpath=data/workplace_assistant \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --resources-server workplace_assistant \ + --config responses_api_models/vllm_model/configs/vllm_model.yaml \ + --output-dir data/workplace_assistant \ + --mode train_preparation \ + --download ``` ### CLI Parameters | Parameter | Required | Description | |-----------|----------|-------------| -| `+config_paths` | Yes | YAML config paths | -| `+output_dirpath` | Yes | Output directory | -| `+mode` | Yes | `example_validation` or `train_preparation` | -| `+should_download` | No | Download missing datasets (default: `false`) | +| `--config` | Yes | YAML config paths (repeatable) | +| `--output-dir` | Yes | Output directory | +| `--mode` | Yes | `example_validation` or `train_preparation` | +| `--download` | No | Download missing datasets (default: `false`) | | `+data_source` | No | `huggingface` (default) or `gitlab` | --- @@ -267,7 +273,7 @@ ng_prepare_data "+config_paths=[resources_servers/workplace_assistant/configs/wo | Missing `responses_create_params` | Sample silently skipped | Add field with valid `input` | | Invalid JSON | Sample skipped | Fix JSON syntax | | Invalid role | Sample skipped | Use `user`, `assistant`, `system`, or `developer` | -| Missing dataset file | `AssertionError` | Create file or set `+should_download=true` | +| Missing dataset file | `AssertionError` | Create file or pass `--download` | Invalid samples are silently skipped. If metrics show fewer examples than expected, check your data format. @@ -305,7 +311,7 @@ with open("your_data.jsonl") as f: ## Validation Process -`ng_prepare_data` performs these steps: +`gym dataset collate` performs these steps: 1. **Load configs** — Parse server configs, identify datasets 2. **Check files** — Verify dataset files exist diff --git a/fern/versions/latest/pages/data/prompt-config.mdx b/fern/versions/latest/pages/data/prompt-config.mdx index ed304f656e..bb998a72a8 100644 --- a/fern/versions/latest/pages/data/prompt-config.mdx +++ b/fern/versions/latest/pages/data/prompt-config.mdx @@ -14,8 +14,8 @@ Apply YAML-based prompt templates at rollout time to build `responses_create_par **In this guide, you will**: 1. Write a prompt config YAML file -2. Apply it during rollout collection with `ng_collect_rollouts` -3. Optionally materialize prompts into JSONL with `ng_materialize_prompts` +2. Apply it during rollout collection with `gym eval run --no-serve` +3. Optionally materialize prompts into JSONL with `gym dataset render` @@ -73,31 +73,32 @@ Literal braces must be doubled (`{{` / `}}`). For example, `\\boxed{{}}` produce ### At rollout time -Pass `+prompt_config=` to `ng_collect_rollouts`: +Pass `--prompt-config ` to `gym eval run --no-serve`: ```bash -ng_collect_rollouts \ - +agent_name=my_agent \ - +input_jsonl_fpath=data/raw_problems.jsonl \ - +output_jsonl_fpath=results/rollouts.jsonl \ - +prompt_config=/path/to/my_prompt.yaml \ - +num_repeats=5 \ - "+responses_create_params={max_output_tokens: 16384, temperature: 1.0}" +gym eval run --no-serve \ + --agent my_agent \ + --input data/raw_problems.jsonl \ + --output results/rollouts.jsonl \ + --prompt-config /path/to/my_prompt.yaml \ + --num-repeats 5 \ + --max-output-tokens 16384 \ + --temperature 1.0 ``` -The `+prompt_config` path must be either an absolute path or a path relative to the Gym repository root. +The `--prompt-config` path must be either an absolute path or a path relative to the Gym repository root. The input JSONL should contain raw fields (e.g. `question`, `expected_answer`) **without** `responses_create_params.input`. The prompt config builds the input messages during rollout collection. ### Standalone materialization -Use `ng_materialize_prompts` to write a prompt template into JSONL without running rollouts: +Use `gym dataset render` to write a prompt template into JSONL without running rollouts: ```bash -ng_materialize_prompts \ - +input_jsonl_fpath=data/raw_problems.jsonl \ - +prompt_config=/path/to/my_prompt.yaml \ - +output_jsonl_fpath=data/materialized.jsonl +gym dataset render \ + --input data/raw_problems.jsonl \ + --prompt-config /path/to/my_prompt.yaml \ + --output data/materialized.jsonl ``` This produces a new JSONL file with `responses_create_params.input` populated from the template. This is useful for inspection or passing to other tools that expect pre-populated input. @@ -119,21 +120,21 @@ Other fields in `responses_create_params` (such as `tools` and `temperature`) ar ## CLI Parameters -### `ng_collect_rollouts` +### `gym eval run --no-serve` | Parameter | Required | Description | |-----------|----------|-------------| -| `+prompt_config` | No | Path to a prompt YAML file. Mutually exclusive with pre-populated `responses_create_params.input` in the JSONL data. | +| `--prompt-config` | No | Path to a prompt YAML file. Mutually exclusive with pre-populated `responses_create_params.input` in the JSONL data. | -See [CLI Commands](/reference/cli-commands) for the full list of `ng_collect_rollouts` parameters. +See [CLI Commands](/reference/cli-commands) for the full list of `gym eval run --no-serve` parameters. -### `ng_materialize_prompts` +### `gym dataset render` | Parameter | Required | Description | |-----------|----------|-------------| -| `+input_jsonl_fpath` | Yes | Raw JSONL data (no `responses_create_params.input`). | -| `+prompt_config` | Yes | Path to prompt YAML file to apply. | -| `+output_jsonl_fpath` | Yes | Output path for materialized JSONL with populated prompts. | +| `--input` | Yes | Raw JSONL data (no `responses_create_params.input`). | +| `--prompt-config` | Yes | Path to prompt YAML file to apply. | +| `--output` | Yes | Output path for materialized JSONL with populated prompts. | --- diff --git a/fern/versions/latest/pages/environment-tutorials/aggregate-metrics.mdx b/fern/versions/latest/pages/environment-tutorials/aggregate-metrics.mdx index f4da3e8793..193c6fab5d 100644 --- a/fern/versions/latest/pages/environment-tutorials/aggregate-metrics.mdx +++ b/fern/versions/latest/pages/environment-tutorials/aggregate-metrics.mdx @@ -9,7 +9,7 @@ After rollout collection, NeMo Gym computes **aggregate metrics** for each agent ## How It Works -1. **Rollouts complete** — `ng_collect_rollouts` gathers verify responses (reward + custom fields) for every task/rollout pair. +1. **Rollouts complete** — `gym eval run --no-serve` gathers verify responses (reward + custom fields) for every task/rollout pair. 2. **Group by agent** — responses are partitioned by agent name. 3. **Call `/aggregate_metrics`** — for each agent, the stripped verify responses are POSTed to the agent's `/aggregate_metrics` endpoint. 4. **Compute stats** — per-task and overall statistics (`mean`, `max`, `min`, `median`, `std`) are computed for every numeric field. If the resources server overrides `compute_metrics()` or `get_key_metrics()`, those are called to add additional metrics. diff --git a/fern/versions/latest/pages/environment-tutorials/integrate-external-environments.mdx b/fern/versions/latest/pages/environment-tutorials/integrate-external-environments.mdx index 514c209347..2362d620b2 100644 --- a/fern/versions/latest/pages/environment-tutorials/integrate-external-environments.mdx +++ b/fern/versions/latest/pages/environment-tutorials/integrate-external-environments.mdx @@ -12,7 +12,7 @@ Integrating external training environments, benchmarks, or agents into a NeMo Gy 1. Because external training environments, benchmarks, or agents include their own rollout orchestration logic (that is, coordinating model and tool calls) and may sometimes return their own reward, NeMo Gym agent servers are the appropriate level to integrate at. 2. You can add a dependency from your NeMo Gym agent server to the third-party library by adding it to the requirements.txt. If your dependency needs are more complicated beyond installing pip packages or GitHub repositories, consider using setup.py and pyproject.toml. 3. After you add a dependency, you can import it in app.py like a normal Python script. -4. You can run the `ng_test` command to run tests on your agent server. Refer to the [ng_test CLI reference](/reference/cli-commands#ng-test-nemo-gym-test) for details. +4. You can run the `gym env test` command to run tests on your agent server. Refer to the [gym env test CLI reference](/reference/cli-commands#gym-env-test) for details. 5. Typically you can just wrap the external library in the `/run` function and omit the `/responses` function. 6. In the `/run` function before calling into the third-party library, you will need to preprocess from NeMo Gym schema into a config for the external library. 7. In the `/run` function after calling into the third-party library, you will need to postprocess from the external library result into a NeMo Gym response. diff --git a/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx b/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx index 743b3a6abe..e141c10cc9 100644 --- a/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx +++ b/fern/versions/latest/pages/environment-tutorials/single-step-environment.mdx @@ -53,7 +53,7 @@ In most cases, the **Resources Server** is where your changes go: define your to Resource servers live in the `resources_servers/` directory. Scaffold a weather server that provides weather information to models: ```bash -ng_init_resources_server +entrypoint=resources_servers/my_weather_tool +gym env init --resources-server my_weather_tool ``` This generates the following structure along with a paired simple agent configuration: @@ -393,7 +393,7 @@ async def test_verify_without_tool_call(server): Run the tests: ```bash -ng_test +entrypoint=resources_servers/my_weather_tool +gym env test --resources-server my_weather_tool ``` For detailed test output: @@ -413,13 +413,12 @@ pytest -v Start the servers: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/my_weather_tool/configs/my_weather_tool.yaml" - -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --resources-server my_weather_tool ``` -`ng_run` reads the config files and starts all three components from the architecture diagram: +`gym env start` reads the config files and starts all three components from the architecture diagram: 1. **Agent Server** (`my_weather_tool_simple_agent`) — the `simple_agent` that orchestrates the seed → model → tool → verify loop 2. **Model Server** (`openai_model`) — proxies LLM inference requests to the OpenAI API @@ -475,12 +474,13 @@ Before training, you collect rollouts to validate that your environment works en With your servers still running, collect rollouts against your example inputs: ```bash -ng_collect_rollouts +agent_name=my_weather_tool_simple_agent \ - +input_jsonl_fpath=resources_servers/my_weather_tool/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/my_weather_tool/data/example_rollouts.jsonl \ - +limit=null \ - +num_repeats=null \ - +num_samples_in_parallel=null +gym eval run --no-serve \ + --agent my_weather_tool_simple_agent \ + --input resources_servers/my_weather_tool/data/example.jsonl \ + --output resources_servers/my_weather_tool/data/example_rollouts.jsonl \ + --limit null \ + --num-repeats null \ + --concurrency null ``` @@ -545,7 +545,7 @@ We'd love to see your contributions! Please make sure your PR includes accurate You've learned how to: -- Initialize a resource server with `ng_init_resources_server` +- Initialize a resources server with `gym env init` - Prepare task data in JSONL format - Implement tool endpoints and verification logic - Configure the required `domain` field and wire components together @@ -654,7 +654,7 @@ Check server status and logs: ```bash # View running servers -ng_status +gym env status # For detailed logs, run the server directly: cd resources_servers/my_weather_tool @@ -662,4 +662,4 @@ source .venv/bin/activate python app.py ``` -Server logs appear in the terminal where `ng_run` was executed. +Server logs appear in the terminal where `gym env start` was executed. diff --git a/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx b/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx index 8843c5109a..bb787ed438 100644 --- a/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx +++ b/fern/versions/latest/pages/environment-tutorials/verification-patterns/llm-as-judge.mdx @@ -94,7 +94,7 @@ The resources server config points the judge at the policy model — `judge_mode The config file ships with a `judge_model` block that starts a dedicated judge server. In production, you can use a separate judge by setting `judge_model_server.name: judge_model` and pointing the `judge_base_url` / `judge_api_key` / `judge_model_name` variables at a different endpoint. This lets you use a different model, provider, or quota for the judge. -Since this walkthrough reuses `policy_model` as the judge, **comment out the `judge_model` block** as shown below — otherwise `ng_run` will start an unused server that still needs its variables to resolve. +Since this walkthrough reuses `policy_model` as the judge, **comment out the `judge_model` block** as shown below — otherwise `gym env start` will start an unused server that still needs its variables to resolve. Be sure to **set `judge_model_server.name` to `policy_model`** as well. @@ -200,18 +200,21 @@ If you are building your own LLM-judge server, you will write similar code — t Start the servers: ```bash -ng_run "+config_paths=[resources_servers/over_refusal_detection/configs/over_refusal_detection.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env start \ + --resources-server over_refusal_detection \ + --model-type openai_model ``` In another terminal, collect rollouts against the 5-entry example dataset to confirm the judge call and reward parsing work end-to-end: ```bash -ng_collect_rollouts \ - +agent_name=over_refusal_detection_simple_agent \ - +input_jsonl_fpath=resources_servers/over_refusal_detection/data/example.jsonl \ - +output_jsonl_fpath=/tmp/over_refusal_smoke_test.jsonl \ - +num_repeats=1 \ - "+responses_create_params={max_output_tokens: 1024, temperature: 1.0}" +gym eval run --no-serve \ + --agent over_refusal_detection_simple_agent \ + --input resources_servers/over_refusal_detection/data/example.jsonl \ + --output /tmp/over_refusal_smoke_test.jsonl \ + --num-repeats 1 \ + --max-output-tokens 1024 \ + --temperature 1.0 ``` Inspect the output JSONL to verify that `reward` values are `0.0`, `0.5`, or `1.0` as expected. Once this looks right, scale to larger datasets and higher `num_repeats`. @@ -347,7 +350,7 @@ Other servers apply the same pattern with domain-specific variations. For exampl 2. Add or reuse a **model server** for the judge; reference it from `judge_model_server`. 3. Design **prompts and parseable verdicts**; handle judge failures gracefully. 4. Set **temperature / max tokens** and **concurrency** for your SLA and budget. -5. Smoke-test with `ng_run` and your resources server's **`data/example.jsonl`**, then scale with `ng_collect_rollouts`. +5. Smoke-test with `gym env start` and your resources server's **`data/example.jsonl`**, then scale with `gym eval run --no-serve`. Done looks like: diff --git a/fern/versions/latest/pages/get-started/installation.mdx b/fern/versions/latest/pages/get-started/installation.mdx index bd20b4f646..ff3b5281be 100644 --- a/fern/versions/latest/pages/get-started/installation.mdx +++ b/fern/versions/latest/pages/get-started/installation.mdx @@ -33,7 +33,7 @@ docker pull nvcr.io/nvidia/nemo-rl:v0.5.0.nemotron_3_super ## Verify Installation ```bash -ng_version +gym --version ``` You should see output like: diff --git a/fern/versions/latest/pages/get-started/quickstart.mdx b/fern/versions/latest/pages/get-started/quickstart.mdx index ebcbea311f..819db0fdc9 100644 --- a/fern/versions/latest/pages/get-started/quickstart.mdx +++ b/fern/versions/latest/pages/get-started/quickstart.mdx @@ -31,10 +31,9 @@ Run your agent on a set of tasks and score the results. This example uses a simp NeMo Gym uses local servers to coordinate your model, agent, and task verification. Start them first: ```bash -environment_config="resources_servers/mcqa/configs/mcqa.yaml" -model_config="responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[${environment_config},${model_config}]" +gym env start \ + --resources-server mcqa \ + --model-type openai_model ``` You should see three server instances starting: @@ -52,12 +51,12 @@ In a new terminal, run your agent on a single task to verify everything works: ```bash source .venv/bin/activate -ng_collect_rollouts \ - +agent_name=mcqa_simple_agent \ - +input_jsonl_fpath=resources_servers/mcqa/data/example.jsonl \ - +output_jsonl_fpath=results/mcqa_rollouts.jsonl \ - +limit=5 \ - +num_repeats=1 +gym eval run --no-serve \ + --agent mcqa_simple_agent \ + --input resources_servers/mcqa/data/example.jsonl \ + --output results/mcqa_rollouts.jsonl \ + --limit 5 \ + --num-repeats 1 ``` You should see a progress bar followed by aggregate metrics: @@ -77,7 +76,7 @@ Rollouts: results/mcqa_rollouts.jsonl Aggregate metrics: results/mcqa_rollouts_aggregate_metrics.json ``` -For per-task pass rates, see `ng_reward_profile` in the [CLI Reference](/reference/cli-commands). +For per-task pass rates, see `gym eval profile` in the [CLI Reference](/reference/cli-commands). ## Explore @@ -86,7 +85,7 @@ Now that you have a working setup, explore what's available. NeMo Gym ships with environments across many domains. You can use these existing environments in addition to building your own. ```bash -ng_list_benchmarks +gym list benchmarks ``` ```text @@ -107,11 +106,11 @@ Available benchmarks in NeMo Gym This lists benchmarks with pre-configured agents. For the full set of environments (including training environments), see the [Available Environments](https://github.com/NVIDIA-NeMo/Gym#-available-environments) table. -Every CLI command supports `+h=true` or `+help=true` for detailed usage information: +Every CLI command supports `-h` or `--help` for detailed usage information: ```bash -ng_help -ng_run +help=true +gym --help +gym env start --help ``` ## Next Steps diff --git a/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx b/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx index f467c1316c..ca65afe6fa 100644 --- a/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx +++ b/fern/versions/latest/pages/infrastructure/engineering-notes/system-design.mdx @@ -6,7 +6,7 @@ This section describes how NeMo Gym components interact during startup and execu ## Control Plane: Server Startup -When you run `ng_run`, the system starts up in four phases: +When you run `gym env start`, the system starts up in four phases: ```mermaid %%{init: {'theme': 'default', 'themeVariables': { 'lineColor': '#5c6bc0', 'primaryTextColor': '#333', 'primaryColor': '#e3f2fd', 'secondaryColor': '#f5f5f5'}}}%% @@ -18,10 +18,12 @@ flowchart LR ### Phase 1: Parse CLI -The `ng_run` command uses Hydra to parse command-line arguments. Users specify configuration files via `+config_paths`: +The `gym env start` command uses Hydra to parse command-line arguments. Users specify configuration files via `--config`: ```bash -ng_run "+config_paths=[resources_servers/math/configs/math.yaml, responses_api_models/openai_model/configs/openai_model.yaml]" +gym env start \ + --resources-server math \ + --model-type openai_model ``` ### Phase 2: Load and Merge Configs @@ -45,7 +47,7 @@ Servers are started in two stages: ```mermaid %%{init: {'theme': 'default', 'themeVariables': { 'lineColor': '#5c6bc0', 'primaryTextColor': '#333'}}}%% flowchart TB - Main["Main Process
ng_run"] + Main["Main Process
gym env start"] Head["Head Server
Port 11000"] Model["Model Server
Own venv + port"] Agent["Agent Server
Own venv + port"] @@ -133,7 +135,7 @@ sequenceDiagram ## Data Plane: Rollout Collection -When you run `ng_collect_rollouts`, the system collects training data by executing rollouts in parallel: +When you run `gym eval run --no-serve`, the system collects training data by executing rollouts in parallel: ```mermaid %%{init: {'theme': 'default', 'themeVariables': { 'lineColor': '#5c6bc0', 'primaryTextColor': '#333'}}}%% @@ -174,5 +176,5 @@ sequenceDiagram The client first queries the Head Server to discover server addresses from the global config, then reads input JSONL and dispatches prompts to the Agent. Completed rollouts are written to output JSONL. **Concurrency behavior differs by use case:** -- **Standalone rollout collection** (`ng_collect_rollouts`): A semaphore gates concurrency via `num_samples_in_parallel` to control load. +- **Standalone rollout collection** (`gym eval run --no-serve`): A semaphore gates concurrency via `num_samples_in_parallel` to control load. - **Training framework integration** (e.g., NeMo RL): All requests are sent without gating; the training framework manages concurrency externally. diff --git a/fern/versions/latest/pages/model-server/vllm.mdx b/fern/versions/latest/pages/model-server/vllm.mdx index 45ec76e5e6..61f71f5993 100644 --- a/fern/versions/latest/pages/model-server/vllm.mdx +++ b/fern/versions/latest/pages/model-server/vllm.mdx @@ -11,9 +11,9 @@ VLLMModel provides a Responses API to Chat Completions mapping middleware layer **To use VLLMModel, just change the `responses_api_models/openai_model/configs/openai_model.yaml` in your config paths to `responses_api_models/vllm_model/configs/vllm_model.yaml`!** ```bash -config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --resources-server example_multi_step \ + --model-type vllm_model ``` @@ -116,12 +116,12 @@ vllm serve \ ### Configure NeMo Gym to use the local vLLM server In a second terminal on the same GPU node that was used to spin up the vLLM server, enter the NeMo Gym Python environment, and start the NeMo Gym servers. ```bash -config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[$config_paths]" \ - ++policy_base_url=http://0.0.0.0:10240/v1 \ - ++policy_model_name=Qwen/Qwen3-4B-Thinking-2507 \ - ++policy_api_key=dummy_key +gym env start \ + --resources-server example_multi_step \ + --model-type vllm_model \ + --model-url http://0.0.0.0:10240/v1 \ + --model Qwen/Qwen3-4B-Thinking-2507 \ + --model-api-key dummy_key ``` @@ -137,9 +137,10 @@ Then replace the `policy_base_url=http://0.0.0.0:10240/v1` to point to the hostn ### Run rollout collection In a third terminal on the same GPU node that was used to spin up the vLLM server, enter the NeMo Gym Python environment, and run rollout collection. ```bash -ng_collect_rollouts +agent_name=example_multi_step_simple_agent \ - +input_jsonl_fpath=resources_servers/example_multi_step/data/example.jsonl \ - +output_jsonl_fpath=results/example_multi_step_rollouts.jsonl +gym eval run --no-serve \ + --agent example_multi_step_simple_agent \ + --input resources_servers/example_multi_step/data/example.jsonl \ + --output results/example_multi_step_rollouts.jsonl ``` ## VLLMModel configuration reference diff --git a/fern/versions/latest/pages/reference/cli-commands.mdx b/fern/versions/latest/pages/reference/cli-commands.mdx index d24b69612a..9196081b7a 100644 --- a/fern/versions/latest/pages/reference/cli-commands.mdx +++ b/fern/versions/latest/pages/reference/cli-commands.mdx @@ -1,554 +1,643 @@ --- title: "CLI Commands" -description: "" +description: "Reference for the unified gym CLI: command groups, flags, and migration from the legacy ng_* commands." position: 3 --- -This page documents all available NeMo Gym CLI commands. +This page documents the NeMo Gym command-line interface. - -Each command has both a short form (such as `ng_run`) and a full form (such as `nemo_gym_run`). They are functionally identical. +All functionality is exposed through a single `gym` entry point, organized into command groups (`gym `). `ng` is a drop-in alias for `gym`. Every group and command supports `-h`/`--help`. + +The legacy `ng_*` / `nemo_gym_*` commands (such as `ng_run` or `nemo_gym_collect_rollouts`) still work but are deprecated. Each one prints a notice pointing at its `gym` replacement and then runs it. See [Migrating from the legacy commands](#migrating-from-the-legacy-commands) at the bottom of this page. ## Quick Reference ```bash -# Display help -ng_help - -# Get detailed help for any command -ng_run +help=true -ng_test +h=true +# General +gym --help # list all command groups +gym --version [--json] # print version and system info +ng ... # 'ng' is an alias for 'gym' + +# Discover benchmarks +gym list benchmarks [--json] # list available benchmarks +gym search [--json] # filter the benchmark list by name + +# Datasets +gym dataset upload # upload a prepared dataset to HF (default) or GitLab +gym dataset download # download a dataset from HF (default) or GitLab +gym dataset rm # delete a dataset from GitLab +gym dataset migrate # move a dataset from GitLab to HF +gym dataset render # generate a dataset preview (materialize prompts) +gym dataset collate # validate and collate a dataset + +# Environments +gym env init # scaffold a new resources server +gym env resolve # resolve and print the final merged config +gym env packages # list packages in a server's virtual environment +gym env test # test resources server(s); all of them if none is given +gym env start # start the servers +gym env status # show running servers + +# Evaluation +gym eval prepare # prepare benchmark data and dump it to disk +gym eval run # collate data, start servers, and collect rollouts +gym eval aggregate # merge sharded rollout results +gym eval profile # compute a reward profile from rollouts + +# Contributor helpers +gym dev test # run NeMo Gym's unit tests ``` ---- - -## Server Management - -Commands for running, testing, and managing NeMo Gym servers. - -### `ng_run` / `nemo_gym_run` - -Start NeMo Gym servers for agents, models, and resources. +## Common Options -This command reads configuration from YAML files specified via `+config_paths` and starts all configured servers. The configuration files should define server instances with their entrypoints and settings. +These options are shared across many commands. -**Configuration Parameter** +| Option | Description | +| --- | --- | +| `--config PATH` | Load a Gym config YAML. Repeatable. Maps to `+config_paths=[...]`. | +| `--benchmark NAME` | Select a registered benchmark by name instead of a config path. | +| `--resources-server NAME` | Select a registered resources server by name. | +| `--model-type NAME` | Select a registered model server type by name (such as `openai_model` or `vllm_model`). | +| `--search-dir DIR` | Extra root directory to search for named components. Repeatable. Lets you register your own benchmarks, resources servers, and models. | +| `--json` | Emit machine-readable JSON instead of human-readable output (reporting commands only). | +| `-v`, `--verbose` | Set the logging level to DEBUG. Flows through to spun-up servers. | +| `-h`, `--help` | Show help for any group or command. | -| Parameter | Type | Description | -| --- | --- | --- | -| `config_paths` | List[str] | Paths to YAML configuration files. Specify using Hydra: `+config_paths="[file1.yaml,file2.yaml]"` | + +The `--benchmark`, `--resources-server`, and `--model-type` selectors resolve a component name to its config file for you, so you do not need to know the project's directory layout. If you mistype a name, the CLI suggests the closest match. To point at a config file directly, use `--config ` instead. + -**Example** -```bash -# Start servers with specific configs -config_paths="resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_run "+config_paths=[${config_paths}]" -``` +### Selecting the model server ---- +Commands that need a model (`gym env start`, `gym eval run`) configure it through four flags: -### `ng_test` / `nemo_gym_test` +| Flag | Description | +| --- | --- | +| `--model-type NAME` | The model server type to load (such as `openai_model`, `vllm_model`, or `local_vllm_model`). | +| `--model`, `-m` | The served model identifier: an API model name, an HF id, or a local checkpoint path. Interpreted per `--model-type`. Maps to `policy_model_name`. | +| `--model-url` | Base URL of an existing model server endpoint. Maps to `policy_base_url`. | +| `--model-api-key` | API key for the model server. Maps to `policy_api_key`. | -Test a specific server module by running its pytest suite and optionally validating example data. -**Parameters** +### Hydra overrides (escape hatch) -| Parameter | Type | Description | -| --- | --- | --- | -| `entrypoint` | str | Entrypoint for this command. Must be a relative path with two parts (such as `responses_api_agents/simple_agent`). | -| `should_validate_data` | bool | Whether to validate the example data (examples, metrics, rollouts, and so on) for this server. Default: `False`. | +The `gym` CLI is a thin wrapper over Gym's Hydra config system. Standard flags (`--flag value`) cover the common inputs. Anything not covered by a flag can still be passed as a raw Hydra override using `+key=value` (add a new key) or `++key=value` (add or override an existing key). Unknown overrides are forwarded to Hydra untouched, so advanced config composition is fully functional. -**Example** +When a command needs overrides for keys that have no dedicated flag, you can keep the whole command in Hydra form (config paths included) or mix `--flag` and `+key=value` styles in one invocation: ```bash -ng_test +entrypoint=resources_servers/example_single_tool_call +gym eval run \ + --benchmark aime24 \ + --model-type openai_model \ + ++responses_create_params.reasoning.effort=low \ + +wandb_project=gym-dev ``` --- -### `ng_test_all` / `nemo_gym_test_all` +## General -Run tests for all server modules in the project. +### `gym --help` -**Parameters** - -| Parameter | Type | Description | -| --- | --- | --- | -| `fail_on_total_and_test_mismatch` | bool | Fail if the number of server modules does not match the number with tests. Default: `False`. | - -**Example** +List all command groups and their descriptions. ```bash -ng_test_all +gym --help ``` ---- - -### `ng_dev_test` / `nemo_gym_dev_test` +### `gym --version` -Run core NeMo Gym tests with coverage reporting. Runs pytest with the `--cov` flag. +Print the NeMo Gym version along with Python, key dependency, and system information. -**Example** +| Option | Description | +| --- | --- | +| `--json` | Output version information as JSON. | ```bash -ng_dev_test -``` - ---- - -### `ng_init_resources_server` / `nemo_gym_init_resources_server` +gym --version -Initialize a new resources server with template files and directory structure. - -**Example** - -```bash -ng_init_resources_server +entrypoint=resources_servers/my_server +# Output as JSON +gym --version --json ``` --- -## Data Collection +## Discovery -Commands for collecting verified rollouts for RL training. +Commands for discovering what you can evaluate against. -### `ng_collect_rollouts` / `nemo_gym_collect_rollouts` +### `gym list benchmarks` -Perform a batch of rollout collection. +List the benchmarks available in NeMo Gym, with their domain, agent, and configured number of repeats. -**Parameters** - -| Parameter | Type | Description | -| --- | --- | --- | -| `agent_name` | str | The agent to collect rollouts from. | -| `input_jsonl_fpath` | str | The input data source to use to collect rollouts, in the form of a file path to a JSONL file. | -| `output_jsonl_fpath` | str | The output data JSONL file path. | -| `limit` | Optional[int] | Maximum number of examples to load and take from the input dataset. | -| `num_repeats` | Optional[int] | The number of times to repeat each example to run. Useful if you want to calculate mean@k, such as mean@4 or mean@16. | -| `num_repeats_add_seed` | bool | When num_repeats >1, add a "seed" parameter on the Responses create params. | -| `num_samples_in_parallel` | Optional[int] | Limit the number of concurrent samples running at once. | -| `responses_create_params` | Dict | Overrides for the `responses_create_params`, such as `temperature` and `max_output_tokens`. | - -**Example** +| Option | Description | +| --- | --- | +| `--json` | Output the benchmark list as JSON. | ```bash -ng_collect_rollouts \ - +agent_name=example_single_tool_call_simple_agent \ - +input_jsonl_fpath=weather_query.jsonl \ - +output_jsonl_fpath=weather_rollouts.jsonl \ - +limit=100 \ - +num_repeats=4 \ - +num_samples_in_parallel=10 -``` - -### `ng_e2e_collect_rollouts` / `nemo_gym_e2e_collect_rollouts` - -Spin up all necessary servers and perform a batch of rollout collection using each dataset inside the provided configs. - -**Parameters** +gym list benchmarks -| Parameter | Type | Description | -| --- | --- | --- | -| `output_jsonl_fpath` | str | The output data JSONL file path. | -| `num_samples_in_parallel` | Optional[int] | Limit the number of concurrent samples running at once. | -| `responses_create_params` | Dict | Overrides for the `responses_create_params`, such as `temperature` and `max_output_tokens`. | +# Machine-readable output for scripting +gym list benchmarks --json | jq '.[].name' +``` -**Examples** +### `gym search` -```bash -ng_e2e_collect_rollouts \ - +output_jsonl_fpath=weather_rollouts.jsonl \ - +num_samples_in_parallel=10 -``` +List benchmarks whose name, agent, resources server, dataset, or domain matches a query. -```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/math_with_judge/configs/math_with_judge.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++wandb_project= \ - ++wandb_name= \ - ++wandb_dir= \ - ++output_jsonl_fpath=results/test_e2e_rollout_collection/aime24.jsonl \ - ++split=validation -``` +| Argument / Option | Description | +| --- | --- | +| `QUERY` | Query to match against component names (positional, required). | +| `--json` | Output matches as JSON. | -Example using GPT-OSS 120B remote vLLM endpoint ```bash -experiment_name=rollouts/test_001 -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/math_with_judge/configs/math_with_judge.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - +skip_venv_if_present=true \ - +wandb_project=gym-dev \ - +wandb_name=$(date +%Y%m%d)/$experiment_name \ - ++output_jsonl_fpath=results/$experiment_name.jsonl \ - ++overwrite_metrics_conflicts=true \ - ++split=validation \ - ++policy_model_name=openai/gpt-oss-120b \ - ++policy_api_key=dummy_key \ - ++policy_base_url=http://0.0.0.0:10240/v1 \ - ++responses_create_params.reasoning.effort=low \ - ++responses_create_params.temperature=1.0 \ - ++responses_create_params.top_p=1.0 &> eval_gptoss120b.log & +gym search math +gym search aime --json ``` --- -### `ng_reward_profile` / `nemo_gym_reward_profile` +## Datasets -Computes statistics on rewards and task difficulty for rollouts collected with `ng_collect_rollouts` with `num_repeats` >1. This outputs a new "reward profiled" dataset, where each task in the dataset has metrics like the average reward, standard deviation, min/max, and pass rate. This is useful in filtering tasks before training for difficulty, variance, or creating a curriculum. +Commands for preparing, previewing, and managing datasets. By default dataset transfer commands use HuggingFace; pass `--storage gitlab` to target the GitLab Registry. -**Parameters** +### `gym dataset upload` -| Parameter | Type | Description | -| --- | --- | --- | -| `input_jsonl_fpath` | str | Path to the original task dataset JSONL file. | -| `rollouts_jsonl_fpath` | str | Path to the rollouts file from `ng_collect_rollouts` (must have been run with `num_repeats` >1). | -| `output_jsonl_fpath` | str | Output file path for the reward profiled dataset. | -| `pass_threshold` | Optional[float] | Reward threshold for computing pass rate. If not specified, pass rate metrics are not included. | +Upload a prepared local JSONL dataset to HuggingFace (default) or GitLab. -**Output Fields** - -Each output row contains all original task fields plus: -- `avg_reward`: Average reward across all rollouts -- `std_reward`: Standard deviation of rewards -- `min_reward`: Minimum reward observed -- `max_reward`: Maximum reward observed -- `total_samples`: Number of rollout samples -- `pass_rate`, `pass_rate_total`, `pass_rate_passed`, `pass_threshold`: (Only if `pass_threshold` is specified) - -**Example** +| Option | Description | +| --- | --- | +| `--storage {hf,gitlab}` | Storage backend. Default: `hf`. | +| `--input`, `-i` | Local JSONL file to upload. | +| `--name` | Dataset name. | +| `--revision` | Dataset revision (version). | +| `--split` | Dataset split (HF only). | +| `--create-pr` | Open a pull request with your changes (HF only). | ```bash -ng_reward_profile \ - +input_jsonl_fpath=tasks.jsonl \ - +rollouts_jsonl_fpath=rollouts.jsonl \ - +output_jsonl_fpath=profiled_tasks.jsonl \ - +pass_threshold=1.0 +# Upload to HuggingFace +gym dataset upload \ + --name my_dataset \ + --input data/train.jsonl \ + --revision 0.0.1 + +# Upload to GitLab +gym dataset upload \ + --storage gitlab \ + --name my_dataset \ + --input data/train.jsonl \ + --revision 0.0.1 ``` ---- - -## Data Management - -Commands for preparing and viewing training data. - -### `ng_prepare_data` / `nemo_gym_prepare_data` - -Prepare and validate training data, generating metrics and statistics for datasets. - -**Parameters** +### `gym dataset download` -| Parameter | Type | Description | -| --- | --- | --- | -| `output_dirpath` | str | Directory path where processed datasets and metrics will be saved. | -| `mode` | Literal["train_preparation", "example_validation"] | Processing mode. Use `train_preparation` to prepare train and validation datasets for training, or `example_validation` to validate example data for PR submission. | -| `should_download` | bool | Whether to automatically download missing datasets from remote registries. Default: `False`. | -| `overwrite_metrics_conflicts` | bool | Whether or not to overwrite metrics conflicts. Default: `False`. | +Download a dataset from HuggingFace (default) or GitLab. -**Example** +| Option | Description | +| --- | --- | +| `--storage {hf,gitlab}` | Storage backend. Default: `hf`. | +| `--repo-id` | HF repo id, such as `org/dataset` (HF only). | +| `--name` | Dataset name (GitLab only). | +| `--revision` | Dataset version (GitLab only). | +| `--artifact` | Remote file to fetch (GitLab: required; HF: optional raw file). | +| `--output`, `-o` | Local destination file. | +| `--output-dir` | Local destination directory; needed when downloading all splits (HF only). | +| `--split` | Dataset split (HF only). | ```bash -config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/example_multi_step \ - +mode=example_validation +# Download a single file from HuggingFace +gym dataset download --repo-id NVIDIA/NeMo-Gym-Math-example_multi_step-v1 \ + --artifact train.jsonl \ + --output data/train.jsonl + +# Download from GitLab +gym dataset download \ + --storage gitlab \ + --name example_multi_step \ + --revision 0.0.1 \ + --artifact train.jsonl \ + --output data/train.jsonl ``` ---- - -## Dataset Registry - GitLab - -Commands for uploading, downloading, and managing datasets in GitLab Model Registry. - -### `ng_upload_dataset_to_gitlab` / `nemo_gym_upload_dataset_to_gitlab` +### `gym dataset rm` -Upload a local JSONL dataset artifact to GitLab. +Delete a dataset from the GitLab Registry. Prompts for confirmation. -**Parameters** - -| Parameter | Type | Description | -| --- | --- | --- | -| `dataset_name` | str | The dataset name. | -| `version` | str | The version of this dataset. Must be in the format `x.x.x`. | -| `input_jsonl_fpath` | str | Path to the JSONL file to upload. | - -**Example** +| Option | Description | +| --- | --- | +| `--name` | Name of the dataset to delete. | ```bash -ng_upload_dataset_to_gitlab \ - +dataset_name=example_multi_step \ - +version=0.0.1 \ - +input_jsonl_fpath=data/train.jsonl +gym dataset rm --name old_dataset ``` ---- - -### `ng_download_dataset_from_gitlab` / `nemo_gym_download_dataset_from_gitlab` +### `gym dataset migrate` -Download a JSONL dataset from GitLab Model Registry. +Migrate a JSONL dataset to HuggingFace from GitLab. Use `gym dataset upload` if you do not want automatic GitLab deletion. -**Parameters** - -| Parameter | Type | Description | -| --- | --- | --- | -| `dataset_name` | str | The dataset name. | -| `version` | str | The version of this dataset. Must be in the format `x.x.x`. | -| `artifact_fpath` | str | The filepath to the artifact to download. | -| `output_fpath` | str | Path where the downloaded dataset will be saved. | - -**Example** +| Option | Description | +| --- | --- | +| `--input`, `-i` | Local JSONL file to upload to HF. | +| `--name` | Dataset name. | +| `--revision` | Dataset revision (HF). | +| `--split` | Dataset split. | +| `--create-pr` | Open a pull request to HF dataset with your changes. | ```bash -ng_download_dataset_from_gitlab \ - +dataset_name=example_multi_step \ - +version=0.0.1 \ - +artifact_fpath=train.jsonl \ - +output_fpath=data/train.jsonl +gym dataset migrate \ + --name my_dataset \ + --input data/train.jsonl \ + --revision 0.0.1 ``` ---- - -### `ng_delete_dataset_from_gitlab` / `nemo_gym_delete_dataset_from_gitlab` +### `gym dataset render` -Delete a dataset from GitLab Model Registry. Prompts for confirmation. +Generate a dataset preview by materializing prompts from a raw input file and a prompt template. -**Parameters** - -| Parameter | Type | Description | -| --- | --- | --- | -| `dataset_name` | str | Name of the dataset to delete from GitLab. | - -**Example** +| Option | Description | +| --- | --- | +| `--input`, `-i` | Raw input JSONL file. | +| `--prompt-config` | Prompt template YAML to apply. | +| `--output`, `-o` | Output JSONL file. | ```bash -ng_delete_dataset_from_gitlab +dataset_name=old_dataset +gym dataset render \ + --input raw.jsonl \ + --prompt-config prompt.yaml \ + --output preview.jsonl ``` ---- - -## Dataset Registry - HuggingFace +### `gym dataset collate` -Commands for uploading and downloading datasets to/from HuggingFace Hub. +Validate and collate a dataset, generating metrics and statistics. -### `ng_upload_dataset_to_hf` / `nemo_gym_upload_dataset_to_hf` - -Upload a JSONL dataset to HuggingFace Hub with optional GitLab deletion after successful upload. - -**Parameters** - -| Parameter | Type | Description | -| --- | --- | --- | -| `hf_token` | str | HuggingFace API token for authentication. | -| `hf_organization` | str | HuggingFace organization name where the dataset will be uploaded. | -| `hf_collection_name` | str | HuggingFace collection name for organizing datasets. | -| `hf_collection_slug` | str | Alphanumeric collection slug found at the end of the collection URI. | -| `dataset_name` | str | Name of the dataset. Will be combined with domain and resources server name. | -| `input_jsonl_fpath` | str | Path to the local JSONL file to upload. | -| `resource_config_path` | str | Path to resources server config file. Used to extract domain for naming convention. | -| `hf_dataset_prefix` | str | Prefix prepended to dataset name. Default: `NeMo-Gym`. | -| `delete_from_gitlab` | Optional[bool] | Delete the dataset from GitLab after successful upload to HuggingFace. Default: `False`. | - -**Example** +| Option | Description | +| --- | --- | +| `--config PATH` | Config file to load. Repeatable. | +| `--resources-server NAME` | Load the named resources server config. | +| `--search-dir DIR` | Extra root directory to search for named components. Repeatable. | +| `--mode {train_preparation,example_validation}` | Use `train_preparation` to prepare train/validation datasets, or `example_validation` to validate example data. | +| `--output-dir` | Output directory for the prepared data. | +| `--download` | Download source datasets before collating. | ```bash -resource_config_path="resources_servers/example_multi_step/configs/example_multi_step.yaml" -ng_upload_dataset_to_hf \ - +dataset_name=my_dataset \ - +input_jsonl_fpath=data/train.jsonl \ - +resource_config_path=${resource_config_path} \ - +delete_from_gitlab=true +gym dataset collate \ + --resources-server example_multi_step \ + --output-dir data/example_multi_step \ + --mode example_validation ``` --- -### `ng_download_dataset_from_hf` / `nemo_gym_download_dataset_from_hf` +## Environments -Download a JSONL dataset from HuggingFace Hub to local filesystem. +Commands for developing, running, and inspecting environments (a dataset + agent harness + resources server + model server). -**Parameters** +### `gym env init` -| Parameter | Type | Description | -| --- | --- | --- | -| `output_fpath` | str | Local file path where the downloaded dataset will be saved. | -| `hf_token` | str | HuggingFace API token for authentication. | -| `artifact_fpath` | str | Name of the artifact file to download from the repository. | -| `repo_id` | str | HuggingFace repository ID in format `organization/dataset-name`. | +Scaffold a new resources server with template files (config, app, tests, README, data directory). -**Example** +| Option | Description | +| --- | --- | +| `--resources-server NAME` | Name of the resources server to create. | ```bash -ng_download_dataset_from_hf \ - +repo_id=NVIDIA/NeMo-Gym-Math-example_multi_step-v1 \ - +artifact_fpath=train.jsonl \ - +output_fpath=data/train.jsonl +gym env init --resources-server my_server ``` ---- +### `gym env resolve` -### `ng_gitlab_to_hf_dataset` / `nemo_gym_gitlab_to_hf_dataset` +Resolve the configs, flags, and overrides into a final merged config and print it. Useful for debugging configuration. Secrets are hidden. -Upload a JSONL dataset to HuggingFace Hub and automatically delete from GitLab after successful upload. +| Option | Description | +| --- | --- | +| `--config PATH` | Config file to load. Repeatable. | -This command always deletes the dataset from GitLab after uploading to HuggingFace. Use `ng_upload_dataset_to_hf` if you want optional deletion control. +```bash +# Merge a resources-server config with a model config, apply an override, and print the result +gym env resolve \ + --config resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + ++responses_create_params.temperature=0.6 +``` -**Parameters** +### `gym env packages` -Same as `ng_upload_dataset_to_hf` but `delete_from_gitlab` is not available. This command always deletes. +Each server has its own isolated virtual environment. List the packages installed in a server's environment. -**Example** +| Option | Description | +| --- | --- | +| `--resources-server NAME` | Name of the resources server. | +| `--outdated` | List only outdated packages. | +| `--json` | Output the package list as JSON. | ```bash -resource_config_path="resources_servers/example_multi_step/configs/example_multi_step.yaml" -ng_gitlab_to_hf_dataset \ - +dataset_name=my_dataset \ - +input_jsonl_fpath=data/train.jsonl \ - +resource_config_path=${resource_config_path} -``` +gym env packages --resources-server example_single_tool_call ---- - -## Configuration & Help - -Commands for debugging configuration and getting help. +# Check for outdated packages +gym env packages \ + --resources-server example_single_tool_call \ + --outdated +``` -### `ng_dump_config` / `nemo_gym_dump_config` +### `gym env test` -Display the resolved Hydra configuration for debugging purposes. +Test resource server(s) by running their pytest suite. If no resources server is given, all of them are tested. -**Example** +| Option | Description | +| --- | --- | +| `--resources-server NAME` | Resources server to test. Omit to test all servers. | ```bash -ng_dump_config "+config_paths=[,]" -``` +# Test a single server +gym env test --resources-server example_single_tool_call ---- +# Test all servers +gym env test +``` -### `ng_help` / `nemo_gym_help` +### `gym env start` -Display a list of available NeMo Gym CLI commands. +Start the NeMo Gym servers (agents, models, resources) defined by the provided configs. Reads configuration from YAML files and runs each configured server in its own environment. -**Example** +| Option | Description | +| --- | --- | +| `--config PATH` | Config file to load. Repeatable. | +| `--benchmark NAME` | Load the named benchmark config (start its servers). | +| `--resources-server NAME` | Load the named resources server config. | +| `--model-type NAME` | Load the named model server type config. | +| `--search-dir DIR` | Extra root directory to search for named components. Repeatable. | +| `--model`, `-m` | Served model identifier. See [Selecting the model server](#selecting-the-model-server). | +| `--model-url` | Model server base URL. | +| `--model-api-key` | Model server API key. | ```bash -ng_help +gym env start \ + --resources-server example_single_tool_call \ + --model-type openai_model + +# Start a benchmark's servers +gym env start \ + --benchmark gpqa \ + --model-type vllm_model ``` ---- - -### `ng_version` / `nemo_gym_version` +### `gym env status` -Display NeMo Gym version and system information. +Show all currently running NeMo Gym servers and their health. -**Parameters** +| Option | Description | +| --- | --- | +| `--json` | Output the server list as JSON. | -| Parameter | Type | Description | -| --- | --- | --- | -| `json_format` | bool | Output in JSON format for programmatic use. Default: `False`. Can be specified with `+json=true`. | +```bash +gym env status +``` -**Example** +``` +NeMo Gym Server Status: -```bash -# Display version information -ng_version +[1] ✓ example_single_tool_call (resources_servers/example_single_tool_call) +{ + 'server_type': 'resources_servers', + 'name': 'example_single_tool_call', + 'port': 58117, + 'pid': 89904, + 'uptime_seconds': '0d 0h 0m 41.5s', +} +... -# Output as JSON -ng_version +json=true +3 servers found (3 healthy, 0 unhealthy) ``` --- -### `ng_pip_list` / `nemo_gym_pip_list` +## Evaluation -Each server has its own isolated virtual environment. To inspect the packages: +Commands for running evaluations end to end: prepare data, collect rollouts, aggregate for sharded runs, and profile. -**Parameters** +### `gym eval prepare` -| Parameter | Type | Description | -| --- | --- | --- | -| `entrypoint` | str | The relative entrypoint path to the server directory | -| `format` | Optional[str] | Output format for pip list. Options: 'columns' (default), 'freeze', 'json'. Default: `None`. | -| `outdated` | bool | List outdated packages. Default: `False`. | +Prepare a benchmark's data by running its `prepare.py` script and dump the result to disk. -**Examples** +| Option | Description | +| --- | --- | +| `--config PATH` | Config file to load. Repeatable. | +| `--benchmark NAME` | Load the named benchmark config. | +| `--search-dir DIR` | Extra root directory to search for named components. Repeatable. | ```bash -# List all packages -ng_pip_list +entrypoint=resources_servers/example_single_tool_call +gym eval prepare --benchmark aime24 +``` -# Output as JSON -ng_pip_list +entrypoint=resources_servers/example_single_tool_call +format=json +### `gym eval run` + +Collate data, start the servers, and collect rollouts. This is the main evaluation command. By default it spins up all required servers. Pass `--no-serve` to collect against servers you already started with `gym env start`. + +| Option | Description | +| --- | --- | +| `--config PATH` | Config file to load. Repeatable. | +| `--benchmark NAME` | Load the named benchmark config. | +| `--resources-server NAME` | Load the named resources server config. | +| `--model-type NAME` | Load the named model server type config. | +| `--search-dir DIR` | Extra root directory to search for named components. Repeatable. | +| `--no-serve` | Collect against already-running servers instead of starting them. | +| `--resume` | Resume from cached rollouts instead of recollecting. | +| `--agent`, `-a` | Agent to collect rollouts with. | +| `--input`, `-i` | Input tasks JSONL file. | +| `--output`, `-o` | Output rollouts JSONL file. | +| `--limit` | Maximum number of tasks to run. | +| `--num-repeats` | Number of rollouts per task (for mean@k metrics). | +| `--prompt-config` | Prompt template YAML to apply. | +| `--concurrency` | Maximum number of concurrent samples. | +| `--split` | Dataset split to use (`train`, `validation`, or `benchmark`). | +| `--model`, `-m` | Served model identifier. | +| `--model-url` | Model server base URL. | +| `--model-api-key` | Model server API key. | +| `--temperature` | Sampling temperature. | +| `--top-p` | Nucleus sampling top-p. | +| `--max-output-tokens` | Maximum output tokens. | -# Check for outdated packages -ng_pip_list +entrypoint=resources_servers/example_single_tool_call +outdated=true +```bash +# End-to-end: spin up servers, then collect rollouts for a benchmark +gym eval run --benchmark aime24 \ + --model-type openai_model \ + --output results/aime24.jsonl \ + --split validation \ + --concurrency 10 + +# Against an already-running server, with a remote vLLM endpoint +gym eval run --no-serve \ + --model-type openai_model \ + --resources-server math_with_judge \ + --output results/test_001.jsonl \ + --split validation \ + --model openai/gpt-oss-120b \ + --model-url http://0.0.0.0:10240/v1 \ + --model-api-key dummy_key \ + --temperature 1.0 \ + --top-p 1.0 ``` ---- - -### `ng_status` / `nemo_gym_status` +### `gym eval aggregate` -View all currently running NeMo Gym servers and their health status. +Merge sharded rollout results into a single rollouts file with aggregate metrics. -**Example** +| Option | Description | +| --- | --- | +| `--config PATH` | Config file to load. Repeatable. | +| `--output`, `-o` | Path for the merged rollouts and aggregate-metrics file. | ```bash -ng_status +gym eval aggregate --output results/merged.jsonl +``` -NeMo Gym Server Status: +### `gym eval profile` -[1] ✓ example_single_tool_call (resources_servers/example_single_tool_call) -{ - 'server_type': 'resources_servers', - 'name': 'example_single_tool_call', - 'port': 58117, - 'pid': 89904, - 'uptime_seconds': '0d 0h 0m 41.5s', -} -[2] ✓ example_single_tool_call_simple_agent (responses_api_agents/simple_agent) -{ - 'server_type': 'responses_api_agents', - 'name': 'simple_agent', - 'port': 58118, - 'pid': 89905, - 'uptime_seconds': '0d 0h 0m 41.5s', -} -[3] ✓ policy_model (responses_api_models/openai_model) -{ - 'server_type': 'responses_api_models', - 'name': 'openai_model', - 'port': 58119, - 'pid': 89907, - 'uptime_seconds': '0d 0h 0m 41.5s', -} +Compute a reward profile from collected rollouts. Outputs per-task statistics such as average reward, standard deviation, min/max, and pass rate, useful for filtering tasks before training by difficulty or variance. Requires rollouts collected with `--num-repeats` greater than 1. -3 servers found (3 healthy, 0 unhealthy) +| Option | Description | +| --- | --- | +| `--inputs` | Materialized inputs JSONL fed to rollout collection. | +| `--rollouts` | Rollouts JSONL produced by collection. | +```bash +gym eval profile \ + --inputs materialized_inputs.jsonl \ + --rollouts rollouts.jsonl ``` --- -## Getting Help +## Contributor Helpers + +### `gym dev test` -For detailed help on any command, run it with `+help=true` or `+h=true`: +Run NeMo Gym's core unit tests with coverage reporting. ```bash -ng_run +help=true -ng_collect_rollouts +h=true +gym dev test ``` -This will display all available configuration parameters and their descriptions. - --- -## Re-install Gym and dependencies +## Migrating from the legacy commands + +The legacy `ng_*` and `nemo_gym_*` are deprecated and will be removed in the future release. +Use the tables below to find their `gym` replacement and update your scripts and workflows. + +### Command mapping + +| Legacy command | New command | +| --- | --- | +| `ng_help` | `gym --help` | +| `ng_version` | `gym --version` | +| `ng_list_benchmarks` | `gym list benchmarks` | +| `ng_run` | `gym env start` | +| `ng_status` | `gym env status` | +| `ng_dump_config` | `gym env resolve` | +| `ng_pip_list` | `gym env packages` | +| `ng_init_resources_server` | `gym env init` | +| `ng_test` | `gym env test` | +| `ng_test_all` | `gym env test` (no `--resources-server`) | +| `ng_prepare_benchmark` | `gym eval prepare` | +| `ng_e2e_collect_rollouts` | `gym eval run` | +| `ng_collect_rollouts` | `gym eval run --no-serve` | +| `ng_aggregate_rollouts` | `gym eval aggregate` | +| `ng_reward_profile` | `gym eval profile` | +| `ng_prepare_data` | `gym dataset collate` | +| `ng_materialize_prompts` | `gym dataset render` | +| `ng_upload_dataset_to_hf` | `gym dataset upload` | +| `ng_upload_dataset_to_gitlab` | `gym dataset upload --storage gitlab` | +| `ng_download_dataset_from_hf` | `gym dataset download` | +| `ng_download_dataset_from_gitlab` | `gym dataset download --storage gitlab` | +| `ng_gitlab_to_hf_dataset` | `gym dataset migrate` | +| `ng_delete_dataset_from_gitlab` | `gym dataset rm` | +| `ng_dev_test` | `gym dev test` | +| `ng_reinstall` | `uv sync --extra dev --group docs` | + +### Replacing Hydra overrides with flags + +The most common Hydra overrides now have dedicated flags: + +| Legacy Hydra override | New flag | +| --- | --- | +| `"+config_paths=[a.yaml,b.yaml]"` | `--config a.yaml --config b.yaml` | +| `+agent_name=...` | `--agent`, `-a` | +| `+input_jsonl_fpath=...` | `--input`, `-i` | +| `+output_jsonl_fpath=...` | `--output`, `-o` | +| `+limit=...` | `--limit` | +| `+num_repeats=...` | `--num-repeats` | +| `+num_samples_in_parallel=...` | `--concurrency` | +| `++split=...` | `--split` | +| `++policy_model_name=...` | `--model`, `-m` | +| `++policy_base_url=...` | `--model-url` | +| `++policy_api_key=...` | `--model-api-key` | +| `++responses_create_params.temperature=...` | `--temperature` | +| `++responses_create_params.top_p=...` | `--top-p` | +| `++responses_create_params.max_output_tokens=...` | `--max-output-tokens` | +| `+mode=...` | `--mode` | +| `+output_dirpath=...` | `--output-dir` | +| `+should_download=true` | `--download` | +| `+prompt_config=...` | `--prompt-config` | +| `+resume_from_cache=true` | `--resume` | +| `+dataset_name=...` | `--name` | +| `+repo_id=...` | `--repo-id` | +| `+revision=...` / `+version=...` | `--revision` | +| `+artifact_fpath=...` | `--artifact` | +| `+output_fpath=...` | `--output`, `-o` | +| `+create_pr=true` | `--create-pr` | +| `+outdated=true` | `--outdated` | + +### Common workflows, before and after + ```bash -ng_reinstall +# Start servers +# Before: +ng_run "+config_paths=[resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +# After: +gym env start \ + --resources-server example_single_tool_call \ + --model-type openai_model + +# End-to-end rollout collection +# Before: +config_paths="responses_api_models/openai_model/configs/openai_model.yaml,resources_servers/math_with_judge/configs/math_with_judge.yaml" +ng_e2e_collect_rollouts "+config_paths=[${config_paths}]" \ + ++output_jsonl_fpath=results/aime24.jsonl \ + ++split=validation +# After: +gym eval run \ + --model-type openai_model \ + --resources-server math_with_judge \ + --output results/aime24.jsonl \ + --split validation + +# Collect against already-running servers +# Before: +ng_collect_rollouts +agent_name=example_single_tool_call_simple_agent \ + +input_jsonl_fpath=weather_query.jsonl \ + +output_jsonl_fpath=weather_rollouts.jsonl \ + +num_repeats=4 +num_samples_in_parallel=10 +# After: +gym eval run --no-serve \ + --agent example_single_tool_call_simple_agent \ + --input weather_query.jsonl \ + --output weather_rollouts.jsonl \ + --num-repeats 4 \ + --concurrency 10 + +# Reward profiling +# Before: +ng_reward_profile +input_jsonl_fpath=materialized_inputs.jsonl +rollouts_jsonl_fpath=rollouts.jsonl +# After: +gym eval profile \ + --inputs materialized_inputs.jsonl \ + --rollouts rollouts.jsonl ``` -This will re-install Gym and its dependencies into the currently activated Python virtual environment. + +Run any command with `--help` to see its full set of flags, and `gym --help` to list every group. + diff --git a/fern/versions/latest/pages/reference/configuration.mdx b/fern/versions/latest/pages/reference/configuration.mdx index 713528cc16..e07c4bf0c1 100644 --- a/fern/versions/latest/pages/reference/configuration.mdx +++ b/fern/versions/latest/pages/reference/configuration.mdx @@ -146,31 +146,36 @@ error_on_almost_servers: true # Exit on invalid configs (default: true) ## Command Line Usage -To run servers, use `ng_run`. NeMo Gym uses [Hydra](https://hydra.cc/) for configuration management. +To run servers, use `gym env start`. NeMo Gym uses [Hydra](https://hydra.cc/) for configuration management. ### Loading Configs ```bash # Load one or more config files -ng_run "+config_paths=[config1.yaml,config2.yaml]" +gym env start \ + --config config1.yaml \ + --config config2.yaml # Use paths stored in env.yaml -ng_run "+config_paths=${my_config_paths}" +gym env start "+config_paths=${my_config_paths}" ``` ### Overriding Values ```bash # Override nested values (use dot notation after server ID) -ng_run "+config_paths=[config.yaml]" \ +gym env start \ + --config config.yaml \ +my_server.resources_servers.my_impl.domain=coding # Override policy model -ng_run "+config_paths=[config.yaml]" \ - +policy_model_name=gpt-4o-mini +gym env start --config config.yaml \ + --model gpt-4o-mini # Disable strict validation -ng_run "+config_paths=[config.yaml]" +error_on_almost_servers=false +gym env start \ + --config config.yaml \ + +error_on_almost_servers=false ``` --- diff --git a/fern/versions/latest/pages/reference/faq.mdx b/fern/versions/latest/pages/reference/faq.mdx index de86b3ea2e..bf556fad9a 100644 --- a/fern/versions/latest/pages/reference/faq.mdx +++ b/fern/versions/latest/pages/reference/faq.mdx @@ -39,9 +39,9 @@ You will only need to manually input the `{dataset_name}` portion of the above w To upload to Hugging Face, use the below command: ```bash resource_config_path="resources_servers/multineedle/configs/multineedle.yaml" -ng_upload_dataset_to_hf \ - +dataset_name={your dataset name} \ - +input_jsonl_fpath=data/multineedle_benchmark.jsonl \ +gym dataset upload \ + --name {your dataset name} \ + --input data/multineedle_benchmark.jsonl \ +resource_config_path=${resource_config_path} ``` @@ -51,11 +51,11 @@ By default, the `split` parameter for uploading is set to `train`, which will ru ```bash resource_config_path="resources_servers/multineedle/configs/multineedle.yaml" -ng_gitlab_to_hf_dataset \ - +dataset_name={your dataset name} \ - +input_jsonl_fpath=data/multineedle_benchmark_validation.jsonl \ - +resource_config_path=${resource_config_path} \ - +split=validation +gym dataset migrate \ + --name {your dataset name} \ + --input data/multineedle_benchmark_validation.jsonl \ + --split validation \ + +resource_config_path=${resource_config_path} ``` ## Uploading with Pull Request workflow @@ -64,13 +64,13 @@ When uploading to an organization repository where you don't have direct write a If you want to specify the revision (branch name), you can add the `+revision={your branch name}` flag. Excluding `create_pr` (or setting it to `false`) assumes you are committing to an existing branch. Including it assumes it will be a brand new branch. ```bash -ng_upload_dataset_to_hf \ - +dataset_name=OpenMathReasoning \ - +input_jsonl_fpath=data/validation.jsonl \ +gym dataset upload \ + --name OpenMathReasoning \ + --input data/validation.jsonl \ + --split validation \ + --revision my-branch-name \ + --create-pr \ +resource_config_path=${resource_config_path} \ - +split=validation \ - +create_pr=true \ - +revision=my-branch-name \ +commit_message="Add validation set" \ +commit_description="Includes 545 examples" ``` @@ -89,7 +89,7 @@ The commit_message and commit_description parameters work for both direct pushes You can optionally pass a `+delete_from_gitlab=true` flag to the above command, which will delete the model and all of its artifacts from GitLab. By default, this is set to `False`. ```bash resource_config_path="resources_servers/multineedle/configs/multineedle.yaml" -ng_upload_dataset_to_hf \ +gym dataset upload \ +dataset_name={your dataset name} \ +input_jsonl_fpath=data/multineedle_benchmark.jsonl \ +resource_config_path=${resource_config_path} \ @@ -106,16 +106,16 @@ You can also run the below command which does the same thing without the need fo ```bash resource_config_path="resources_servers/multineedle/configs/multineedle.yaml" -ng_gitlab_to_hf_dataset \ - +dataset_name={your dataset name} \ - +input_jsonl_fpath=data/multineedle_benchmark.jsonl \ +gym dataset migrate \ + --name {your dataset name} \ + --input data/multineedle_benchmark.jsonl \ +resource_config_path=${resource_config_path} ``` If you've already uploaded to Hugging Face and just want to do a standalone delete from GitLab: ```bash -ng_delete_dataset_from_gitlab \ - +dataset_name={your dataset name} +gym dataset rm \ + --name {your dataset name} ``` @@ -128,24 +128,24 @@ Downloading a dataset from Hugging Face is straightforward: **For structured datasets (with train/validation/test splits):** ```bash -ng_download_dataset_from_hf \ - +repo_id=nvidia/Nemotron-RL-knowledge-mcqa \ - +output_dirpath=data/mcqa \ - +split=train +gym dataset download \ + --repo-id nvidia/Nemotron-RL-knowledge-mcqa \ + --output-dir data/mcqa \ + --split train ``` The `split` parameter is optional. If omitted, all available splits will be downloaded as separate JSONL files. **For raw file repositories (with specific JSONL files):** ```bash -ng_download_dataset_from_hf \ - +repo_id=nvidia/Nemotron-RL-instruction_following \ - +output_dirpath=data/instruction_following \ - +artifact_fpath=instruction_following.jsonl +gym dataset download \ + --repo-id nvidia/Nemotron-RL-instruction_following \ + --output-dir data/instruction_following \ + --artifact instruction_following.jsonl ``` Use `artifact_fpath` when the Hugging Face repo contains raw/arbitrary JSONL files rather than structured dataset splits. You cannot specify both `split` and `artifact_fpath`. # How To: Prepare and validate data for PR submission or RL training -When you use `ng_init_resources_server +entrypoint=resources_servers/example_multi_step` to initialize a resources server, you will get a config.yaml that looks like the below code block. The dataset information for training, validation, and example will be inside the scope of your agent config (e.g. under simple_agent) and is a list of dataset objects. +When you use `gym env init --resources-server example_multi_step` to initialize a resources server, you will get a config.yaml that looks like the below code block. The dataset information for training, validation, and example will be inside the scope of your agent config (e.g. under simple_agent) and is a list of dataset objects. ```yaml example_multi_step_resources_server: @@ -220,39 +220,45 @@ A dataset object consists of: Each config.yaml in the resources server requires at least one agent with one example dataset. This example dataset is the first 5 rows of your train dataset that is used for sanity checks on the format for your dataset and the format of each individual example and for others to quickly understand your data. -For every PR that contributes data, we require common dataset statistics and sanity checks on the data itself. This process is also helpful to catch any simple issues before you ever train with NeMo RL. NeMo Gym provides a helper command ng_prepare_data to do so. +For every PR that contributes data, we require common dataset statistics and sanity checks on the data itself. This process is also helpful to catch any simple issues before you ever train with NeMo RL. NeMo Gym provides a helper command gym dataset collate to do so. ```bash -config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_prepare_data "+config_paths=[$config_paths]" \ - +output_dirpath=data/example_multi_step \ - +mode=example_validation +gym dataset collate \ + --resources-server example_multi_step \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/example_multi_step \ + --mode example_validation ``` -To download missing datasets automatically, add +should_download=true. By default, datasets are downloaded from Hugging Face: +To download missing datasets automatically, add --download. By default, datasets are downloaded from Hugging Face: ```bash -ng_prepare_data "+config_paths=[$config_paths]" \ - +output_dirpath=data/example_multi_step \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --resources-server example_multi_step \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/example_multi_step \ + --mode train_preparation \ + --download ``` For NVIDIA internal users, you can download from GitLab instead: ```bash -ng_prepare_data "+config_paths=[$config_paths]" \ - +output_dirpath=data/example_multi_step \ - +mode=train_preparation \ - +should_download=true \ +gym dataset collate \ + --resources-server example_multi_step \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/example_multi_step \ + --mode train_preparation \ + --download \ +data_source=gitlab ``` Run NeMo Gym servers the exact same way with the same configs! ```bash -ng_run "+config_paths=[$config_paths]" +gym env start \ + --resources-server example_multi_step \ + --model-type openai_model ``` -The `ng_prepare_data` command will: +The `gym dataset collate` command will: 1. Attempt to load all the datasets you specified from disk. Missing datasets will be reported before any processing is done. 2. For each dataset, read example by example. Check the format and report the filepaths and indices/ranges of offending examples if any. 1. We only require that the dataset has one key responses_create_params which is valid Responses API schema. @@ -267,15 +273,15 @@ The `ng_prepare_data` command will: 4. Check that the aggregate statistics of individual datasets match those of existing aggregate statistics. 5. Collate all the examples into one final train and validation dataset jsonl files at the output dirpath specified for downstream NeMo RL or other train framework consumption. 6. The final aggregate statistics are reported and saved next to the train and validation datasets. -7. [NeMo RL train] Use the exact same config paths to ng_prepare_data and the train/validation dataset paths output in step 5. There is no special pre or post processing done in the NeMo Gym/RL integration other than shuffling and distributed data loading. What you see is what you get. +7. [NeMo RL train] Use the exact same config paths to gym dataset collate and the train/validation dataset paths output in step 5. There is no special pre or post processing done in the NeMo Gym/RL integration other than shuffling and distributed data loading. What you see is what you get. -The `ng_prepare_data` command has 2 modes, one for actual train and validation set preparation, and one for example validation intended to sanity check your data format. You would typically run `+mode=example_validation` when first contributing a resources server, and then run with `+mode=train_preparation` when you actually go to train. +The `gym dataset collate` command has 2 modes, one for actual train and validation set preparation, and one for example validation intended to sanity check your data format. You would typically run `--mode example_validation` when first contributing a resources server, and then run with `--mode train_preparation` when you actually go to train. ```bash -config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_prepare_data "+config_paths=[$config_paths]" \ - +output_dirpath=data/example_multi_step \ - +mode=example_validation +gym dataset collate \ + --resources-server example_multi_step \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/example_multi_step \ + --mode example_validation ``` # How To: Profile your resources server @@ -285,23 +291,24 @@ In one terminal, start your agent, model, and resources servers, with profiling - `profiling_enabled` (bool): whether profiling is enabled or not. By default this is disabled since it incurs some slight overhead we don't want at runtime. - `profiling_results_dirpath` (str): The directory to save all server profiling results in. Previous logs for the same will be overwritten in the same directory. ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/math_with_judge/configs/bytedtsinghua_dapo17k.yaml" -ng_run "+config_paths=[${config_paths}]" \ +gym env start \ + --model-type openai_model \ + --config resources_servers/math_with_judge/configs/bytedtsinghua_dapo17k.yaml \ +profiling_enabled=true \ +profiling_results_dirpath=results/profiling/math_with_judge ``` In another terminal, run some large number of rollouts against your servers. Use the `limit` and `num_repeats` flags to adjust the number of samples you want to run. ```bash -ng_collect_rollouts +agent_name=math_with_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/math_with_judge/data/dapo17k_bytedtsinghua_train.jsonl \ - +output_jsonl_fpath=temp/math_with_judge_rollouts.jsonl \ - +limit=1024 \ - +num_repeats=1 +gym eval run --no-serve \ + --agent math_with_judge_simple_agent \ + --input resources_servers/math_with_judge/data/dapo17k_bytedtsinghua_train.jsonl \ + --output temp/math_with_judge_rollouts.jsonl \ + --limit 1024 \ + --num-repeats 1 ``` -After `ng_collect_rollouts` finishes, ctrl+c to quit your servers. You should see some output in the terminal like the following: +After `gym eval run` finishes, ctrl+c to quit your servers. You should see some output in the terminal like the following: ```bash ``` @@ -334,7 +341,7 @@ NeMo Gym automatically sets up Ray for distributed computing for CPU-intensive t Ray is initialized when you start NeMo Gym servers: ```bash -ng_run "+config_paths=[$config_paths]" +gym env start --config $config_paths ``` The initialization happens in two places: @@ -393,7 +400,7 @@ Tying back to NeMo Gym, NeMo Gym can be used to create synthetic data for SFT tr # FAQ: PermissionError when starting NeMo Gym in sandboxed environments -If you see an error like the following when running `ng_run`: +If you see an error like the following when running `gym env start`: ```python PermissionError: [Errno 1] Operation not permitted (originated from sysctl() malloc 1/3) @@ -424,7 +431,7 @@ If you're running NeMo Gym in a sandboxed environment and hit this error, you'll 2. **Grant additional permissions** to allow Ray to access process information 3. **Run outside the sandbox** in a normal terminal environment -For most development and production use cases, simply running `ng_run` in your regular terminal will work without any issues. +For most development and production use cases, simply running `gym env start` in your regular terminal will work without any issues. **Specific workaround for Cursor AI:** diff --git a/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx b/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx index 535b42592c..26a170e515 100644 --- a/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx +++ b/fern/versions/latest/pages/training-tutorials/multi-environment-training.mdx @@ -13,24 +13,24 @@ Suppose you want to use both the example_single_tool_call and example_multi_step For example_single_tool_call: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --model-type openai_model \ + --resources-server example_single_tool_call ``` For example_multi_step: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/example_multi_step/configs/example_multi_step.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --resources-server example_multi_step ``` To use both environments, add the YAML configs together as follows: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,\ -resources_servers/example_multi_step/configs/example_multi_step.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --resources-server example_single_tool_call \ + --resources-server example_multi_step ``` ## Dataset Preparation @@ -45,9 +45,9 @@ jq -c '. + {"agent_ref": {"type": "responses_api_agents", "name": "example_multi Run rollout collection as usual. ```bash -ng_collect_rollouts \ - +input_jsonl_fpath=results/test_multiverifier_input.jsonl \ - +output_jsonl_fpath=results/test_multiverifier_outputs.jsonl +gym eval run --no-serve \ + --input results/test_multiverifier_input.jsonl \ + --output results/test_multiverifier_outputs.jsonl ``` Inside `results/test_multiverifier_outputs.jsonl`, you should see 10 rows with appropriate responses for each row. diff --git a/fern/versions/latest/pages/training-tutorials/nemo-rl-grpo/setup.mdx b/fern/versions/latest/pages/training-tutorials/nemo-rl-grpo/setup.mdx index 77cb088a08..a3cc2cb6aa 100644 --- a/fern/versions/latest/pages/training-tutorials/nemo-rl-grpo/setup.mdx +++ b/fern/versions/latest/pages/training-tutorials/nemo-rl-grpo/setup.mdx @@ -205,7 +205,7 @@ uv run python -c "import ray; ray.shutdown()" **Estimated time**: ~5 minutes -The Workplace Assistant dataset must be downloaded from HuggingFace and prepared for training. This runs `ng_prepare_data` to download and validate the dataset, and to add an `agent_ref` property to each example that tells NeMo Gym which agent server should handle that example. +The Workplace Assistant dataset must be downloaded from HuggingFace and prepared for training. This runs `gym dataset collate` to download and validate the dataset, and to add an `agent_ref` property to each example that tells NeMo Gym which agent server should handle that example. Clone and setup the Gym Python environment: @@ -226,13 +226,12 @@ echo "hf_token: {your HF token}" >> env.yaml Prepare the data: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/workplace_assistant/configs/workplace_assistant.yaml" - -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/workplace_assistant \ - +mode=train_preparation \ - +should_download=true \ +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resources-server workplace_assistant \ + --output-dir data/workplace_assistant \ + --mode train_preparation \ + --download \ +data_source=huggingface ``` diff --git a/fern/versions/latest/pages/training-tutorials/offline-training-w-rollouts.mdx b/fern/versions/latest/pages/training-tutorials/offline-training-w-rollouts.mdx index 46218bca15..dbaddd3ddf 100644 --- a/fern/versions/latest/pages/training-tutorials/offline-training-w-rollouts.mdx +++ b/fern/versions/latest/pages/training-tutorials/offline-training-w-rollouts.mdx @@ -267,9 +267,10 @@ Test your improved model by generating new rollouts on held-out evaluation tasks ```bash # Generate rollouts with improved model -ng_collect_rollouts +agent_name=improved_agent \ - +input_jsonl_fpath=evaluation_tasks.jsonl \ - +output_jsonl_fpath=post_training_rollouts.jsonl +gym eval run --no-serve \ + --agent improved_agent \ + --input evaluation_tasks.jsonl \ + --output post_training_rollouts.jsonl ``` Compare key metrics like average reward, success rate, and task-specific performance against your baseline to measure improvement. diff --git a/fern/versions/latest/pages/training-tutorials/verl.mdx b/fern/versions/latest/pages/training-tutorials/verl.mdx index b44cada8ec..e91f43bc74 100644 --- a/fern/versions/latest/pages/training-tutorials/verl.mdx +++ b/fern/versions/latest/pages/training-tutorials/verl.mdx @@ -36,14 +36,12 @@ Using NeMo Gym, prepare the training dataset for your environment. Each row need cd $NEMO_GYM_ROOT source .venv/bin/activate -config_paths="resources_servers/workplace_assistant/configs/workplace_assistant.yaml,\ -responses_api_models/vllm_model/configs/vllm_model_for_training.yaml" - -ng_prepare_data \ - "+config_paths=[${config_paths}]" \ - +output_dirpath=data/workplace_assistant \ - +mode=train_preparation \ - +should_download=true \ +gym dataset collate \ + --resources-server workplace_assistant \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --output-dir data/workplace_assistant \ + --mode train_preparation \ + --download \ +data_source=huggingface ``` diff --git a/fern/versions/latest/pages/troubleshooting/configuration.mdx b/fern/versions/latest/pages/troubleshooting/configuration.mdx index a2a303290c..29efd009b8 100644 --- a/fern/versions/latest/pages/troubleshooting/configuration.mdx +++ b/fern/versions/latest/pages/troubleshooting/configuration.mdx @@ -3,7 +3,7 @@ title: "Configuration Errors" description: "" position: 1 --- -These errors appear when running `ng_run` or `ng_collect_rollouts`. NeMo Gym validates configuration at startup before launching servers. +These errors appear when running `gym env start` or `gym eval run --no-serve`. NeMo Gym validates configuration at startup before launching servers. See the [Configuration reference](/reference/configuration) for complete configuration syntax and options. @@ -39,7 +39,9 @@ policy_api_key: sk-your-api-key Or pass via command line: ```bash -ng_run "+config_paths=[config.yaml]" +policy_api_key=sk-your-api-key +gym env start \ + --config config.yaml \ + --model-api-key sk-your-api-key ``` ### Server Reference Not Found @@ -54,15 +56,18 @@ names the referencing instance/field and, when a similarly named server exists, **Common causes**: - Typo in the server name -- Referenced server's config file not included in `+config_paths` +- Referenced server's config file not included in the `--config` flags - Server defined in a different config file that wasn't loaded **Fix**: 1. Check server name spelling in your config -2. Ensure all required config files are in `+config_paths`: +2. Ensure all required config files are passed with `--config`: ```bash -ng_run "+config_paths=[model.yaml,resource.yaml,agent.yaml]" +gym env start \ + --config model.yaml \ + --config resource.yaml \ + --config agent.yaml ``` --- @@ -104,7 +109,9 @@ error_on_almost_servers: false ```bash # Or via command line -ng_run "+config_paths=[config.yaml]" +error_on_almost_servers=false +gym env start \ + --config config.yaml \ + +error_on_almost_servers=false ``` diff --git a/nemo_gym/benchmarks.py b/nemo_gym/benchmarks.py index 9ba1e9fafd..5a598b62e2 100644 --- a/nemo_gym/benchmarks.py +++ b/nemo_gym/benchmarks.py @@ -14,26 +14,19 @@ # limitations under the License. """Benchmark discovery and preparation utilities.""" -import importlib -from glob import glob -from multiprocessing import Pool from pathlib import Path -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional -import rich from omegaconf import DictConfig, OmegaConf -from pydantic import BaseModel, Field -from rich.table import Table -from tqdm.auto import tqdm +from pydantic import BaseModel from nemo_gym import PARENT_DIR -from nemo_gym.config_types import BaseNeMoGymCLIConfig, BenchmarkDatasetConfig +from nemo_gym.config_types import BenchmarkDatasetConfig from nemo_gym.global_config import ( POLICY_MODEL_KEY_NAME, GlobalConfigDictParser, GlobalConfigDictParserConfig, get_first_server_config_dict, - get_global_config_dict, ) @@ -107,177 +100,16 @@ def _load_benchmarks_from_config_paths(config_paths: List[Path]) -> Dict[str, Be return benchmarks_dict -def list_benchmarks() -> None: - """CLI command: list available benchmarks.""" - global_config_dict = get_global_config_dict( - global_config_dict_parser_config=GlobalConfigDictParserConfig( - initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, - ) - ) - BaseNeMoGymCLIConfig.model_validate(global_config_dict) - - assert BENCHMARKS_DIR.exists(), "Missing benchmarks directory" - - config_paths = glob("**/config.yaml", root_dir=BENCHMARKS_DIR, recursive=True) - config_paths = [BENCHMARKS_DIR / p for p in config_paths] - config_paths = sorted(config_paths) - - benchmarks = _load_benchmarks_from_config_paths(config_paths) - - if not benchmarks: - rich.print("[yellow]No benchmarks found.[/yellow]") - rich.print(f"Expected benchmarks directory: {BENCHMARKS_DIR}") - return - - table = Table(title=f"Available benchmarks in NeMo Gym ({len(benchmarks)})") - table.add_column("Benchmark name") - table.add_column("Agent name") - table.add_column("Num repeats") - - for name, bench in benchmarks.items(): - table.add_row(name, bench.agent_name, str(bench.num_repeats)) - - rich.print(table) - - -class PrepareBenchmarkConfig(BaseNeMoGymCLIConfig): - """ - Prepare benchmark data by running the benchmark's prepare.py script. - - The benchmark is identified from a config_paths entry pointing to a - benchmarks/*/config.yaml file. - - Examples: - - ```bash - ng_prepare_benchmark "+config_paths=[benchmarks/aime24/config.yaml]" - ``` - """ - - use_cached_prepared_benchmarks: bool = Field( - default=False, description="Skip benchmark preparation if the prepared file is already present" - ) - num_prepare_benchmark_processes: int = Field( - default=1, description="Number of processes to parallelize benchmark preparation" - ) - - -def _multiprocess_benchmark_prepare_fn(args): - benchmark_config: BenchmarkConfig - prepare_module_path: str - (benchmark_config, prepare_module_path) = args - - print(f"Preparing benchmark: {benchmark_config.name}") - - module = importlib.import_module(prepare_module_path) - output_fpath = module.prepare() - assert output_fpath.absolute() == benchmark_config.dataset.jsonl_fpath.absolute(), ( - f"Expected the actual prepared dataset output fpath to match the jsonl_fpath set in the config. Instead got {output_fpath=} jsonl_fpath={benchmark_config.dataset.jsonl_fpath}" - ) - print(f"Benchmark data prepared at: {output_fpath}") - - -def prepare_benchmark() -> None: - """CLI command: prepare benchmark data.""" - global_config_dict = get_global_config_dict( - global_config_dict_parser_config=GlobalConfigDictParserConfig( - initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, - ) - ) - prepare_benchmark_config = PrepareBenchmarkConfig.model_validate(global_config_dict) - - benchmarks_dict: Dict[str, BenchmarkConfig] = dict() - for server_instance_name in global_config_dict: - server_config = global_config_dict[server_instance_name] - if not isinstance(server_config, (dict, DictConfig)) or "responses_api_agents" not in server_config: - continue - - inner_server_config = get_first_server_config_dict(global_config_dict, server_instance_name) - - datasets: List[BenchmarkDatasetConfig] = [] - for dataset in inner_server_config.get("datasets") or []: - if dataset["type"] != "benchmark": - continue +# Backward-compatibility shims (CLI refactor): these symbols moved to `nemo_gym.cli.eval`. +# Re-exported lazily to avoid a circular import; accessing them emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 - datasets.append(BenchmarkDatasetConfig.model_validate(dataset)) - if len(datasets) < 1: - continue - - assert len(datasets) == 1, ( - f"Expected 1 benchmark dataset for `{server_instance_name}`, but found {len(datasets)}!" - ) - - dataset = datasets[0] - - benchmarks_dict[server_instance_name] = BenchmarkConfig( - name=dataset.name, - path=Path(""), - agent_name=server_instance_name, - num_repeats=dataset.num_repeats, - dataset=dataset, - ) - - assert benchmarks_dict, ( - 'No benchmark config found in config_paths. Pass a benchmark config, e.g.: "+config_paths=[benchmarks/aime24/config.yaml]"' - ) - - # Validate all benchmarks before preparing any - prepare_script_missing: List[BenchmarkConfig] = [] - prepare_function_missing: List[BenchmarkConfig] = [] - - validated: List[Tuple[BenchmarkConfig, str]] = [] - already_prepared: List[BenchmarkConfig] = [] - for benchmark_config in benchmarks_dict.values(): - prepare_script_path = benchmark_config.dataset.prepare_script - if not prepare_script_path.exists(): - prepare_script_missing.append(benchmark_config) - continue - - prepare_module_path = ".".join(prepare_script_path.with_suffix("").parts) - module = importlib.import_module(prepare_module_path) - if not hasattr(module, "prepare"): - prepare_function_missing.append(benchmark_config) - continue - - is_already_prepared = benchmark_config.dataset.jsonl_fpath.exists() - if prepare_benchmark_config.use_cached_prepared_benchmarks and is_already_prepared: - already_prepared.append(benchmark_config) - continue - - validated.append((benchmark_config, prepare_module_path)) - - if already_prepared: - already_prepared_str = "".join(f"- {bc.name}: {bc.dataset.jsonl_fpath}\n" for bc in already_prepared) - already_prepared_str = f"""The following benchmarks have already been prepared. Since `use_cached_prepared_benchmarks=true`, we will skip re-preparation of those benchmarks. - {already_prepared_str}""" - print(already_prepared_str) - - errors_to_print = "" - if prepare_script_missing: - prepare_script_missing_str = "".join( - f"- {bc.name}: {bc.dataset.prepare_script}\n" for bc in prepare_script_missing - ) - errors_to_print += f"""The following benchmarks are missing a valid prepare script: -{prepare_script_missing_str} -""" - if prepare_function_missing: # pragma: no cover - prepare_function_missing_str = "".join( - f"- {bc.name}: {bc.dataset.prepare_script}\n" for bc in prepare_function_missing - ) - errors_to_print += f"""The following benchmarks have a prepare script, but are missing the prepare function: -{prepare_function_missing_str} -""" - if errors_to_print: - errors_to_print = f"""Did not prepare any benchmarks due to benchmark config errors. -{errors_to_print}""" - raise RuntimeError(errors_to_print) - - # Prepare after all validations pass - if prepare_benchmark_config.num_prepare_benchmark_processes > 1: # pragma: no cover - with Pool(processes=prepare_benchmark_config.num_prepare_benchmark_processes) as pool: - results = pool.imap_unordered(_multiprocess_benchmark_prepare_fn, validated) - list(tqdm(results, total=len(validated))) - else: - results = map(_multiprocess_benchmark_prepare_fn, validated) - list(tqdm(results, total=len(validated))) +__getattr__ = moved_attr_getter( + __name__, + { + "list_benchmarks": "nemo_gym.cli.eval", + "PrepareBenchmarkConfig": "nemo_gym.cli.eval", + "prepare_benchmark": "nemo_gym.cli.eval", + }, +) diff --git a/nemo_gym/cli/__init__.py b/nemo_gym/cli/__init__.py new file mode 100644 index 0000000000..43f54bc668 --- /dev/null +++ b/nemo_gym/cli/__init__.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_gym.cli._compat import moved_attr_getter + + +# The single `nemo_gym/cli.py` module was split into the `nemo_gym.cli` package. Its public +# surface is re-exported lazily from the new submodules so existing `from nemo_gym.cli import X` +# imports (e.g. NeMo-RL's `RunHelper` / `GlobalConfigDictParserConfig`) keep working. Lazy +# resolution keeps `import nemo_gym.cli` cheap — it doesn't eagerly pull in hydra, wandb or ray — +# and avoids circular imports. Each access emits a DeprecationWarning pointing at the new path. +__getattr__ = moved_attr_getter( + __name__, + { + # Server orchestration + run/test entry points (old `nemo_gym/cli.py`) + "RunHelper": "nemo_gym.cli.env", + "RunConfig": "nemo_gym.cli.env", + "TestConfig": "nemo_gym.cli.env", + "TestAllConfig": "nemo_gym.cli.env", + "PipListConfig": "nemo_gym.cli.env", + "run": "nemo_gym.cli.env", + "test": "nemo_gym.cli.env", + "test_all": "nemo_gym.cli.env", + "init_resources_server": "nemo_gym.cli.env", + "dump_config": "nemo_gym.cli.env", + "status": "nemo_gym.cli.env", + "pip_list": "nemo_gym.cli.env", + "e2e_rollout_collection": "nemo_gym.cli.eval", + "dev_test": "nemo_gym.cli.dev", + "VersionConfig": "nemo_gym.cli.general", + "version": "nemo_gym.cli.general", + # Never actually lived in cli; reachable via the old module's top-level import. + "GlobalConfigDictParserConfig": "nemo_gym.global_config", + }, +) diff --git a/nemo_gym/cli/_compat.py b/nemo_gym/cli/_compat.py new file mode 100644 index 0000000000..f3c6934796 --- /dev/null +++ b/nemo_gym/cli/_compat.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Backward-compatibility helpers for symbols relocated during the CLI refactor. + +It provides a lazy import mechanism for symbols that were moved to the `nemo_gym.cli` +package and emits a `DeprecationWarning` pointing at the correct import path. + +Note that an eager re-export `from nemo_gym.cli.x import foo` at the top of a +core module would create a circular import. +""" + +import importlib +import warnings +from typing import Any, Callable, Mapping + + +def moved_attr_getter(module_name: str, moved: Mapping[str, str]) -> Callable[[str], Any]: + """Build a module `__getattr__` that re-exports relocated symbols with a deprecation warning. + + Args: + module_name: The `__name__` of the module installing the shim (the old location). + moved: Maps each old attribute name to its new location. The value is either + `"package.module"` (same attribute name) or `"package.module:new_name"` + when the symbol was also renamed. + + Returns: + A function suitable for assigning to a module's `__getattr__`. + """ + + def __getattr__(name: str) -> Any: + target = moved.get(name) + if target is None: + raise AttributeError(f"module {module_name!r} has no attribute {name!r}") + new_module, _, new_name = target.partition(":") + new_name = new_name or name + warnings.warn( + f"`{module_name}.{name}` has moved to `{new_module}.{new_name}` and will be removed in a " + f"future release. Update your import to `from {new_module} import {new_name}`.", + DeprecationWarning, + stacklevel=2, + ) + return getattr(importlib.import_module(new_module), new_name) + + return __getattr__ diff --git a/nemo_gym/cli/dataset.py b/nemo_gym/cli/dataset.py new file mode 100644 index 0000000000..bb2d211d50 --- /dev/null +++ b/nemo_gym/cli/dataset.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_gym.config_types import ( + DeleteJsonlDatasetGitlabConfig, + DownloadJsonlDatasetGitlabConfig, + DownloadJsonlDatasetHuggingFaceConfig, + UploadJsonlDatasetGitlabConfig, + UploadJsonlDatasetHuggingFaceConfig, + UploadJsonlDatasetHuggingFaceMaybeDeleteConfig, +) +from nemo_gym.dataset_orchestrator import ( + delete_jsonl_dataset_from_gitlab, + upload_jsonl_dataset_to_hf_maybe_delete, +) +from nemo_gym.gitlab_utils import download_jsonl_dataset, upload_jsonl_dataset +from nemo_gym.global_config import GlobalConfigDictParserConfig, get_global_config_dict +from nemo_gym.hf_utils import download_hf_dataset_as_jsonl +from nemo_gym.prompt import MaterializePromptsConfig, materialize_prompts +from nemo_gym.train_data_utils import TrainDataProcessor + + +def upload_jsonl_dataset_cli() -> None: # pragma: no cover + global_config = get_global_config_dict() + config = UploadJsonlDatasetGitlabConfig.model_validate(global_config) + upload_jsonl_dataset(config) + + +def download_jsonl_dataset_cli() -> None: # pragma: no cover + global_config = get_global_config_dict() + config = DownloadJsonlDatasetGitlabConfig.model_validate(global_config) + download_jsonl_dataset(config) + + +def upload_jsonl_dataset_to_hf_cli() -> None: # pragma: no cover + global_config = get_global_config_dict() + config = UploadJsonlDatasetHuggingFaceMaybeDeleteConfig.model_validate(global_config) + upload_jsonl_dataset_to_hf_maybe_delete(config, delete_from_gitlab=config.delete_from_gitlab) + + +def download_jsonl_dataset_from_hf_cli() -> None: # pragma: no cover + global_config = get_global_config_dict() + config = DownloadJsonlDatasetHuggingFaceConfig.model_validate(global_config) + + if config.artifact_fpath: + print(f"Downloading file '{config.artifact_fpath}' from '{config.repo_id}'...") + else: + print(f"Downloading '{config.split or 'all'}' split(s) from '{config.repo_id}'...") + + download_hf_dataset_as_jsonl(config) + + +def delete_jsonl_dataset_from_gitlab_cli() -> None: # pragma: no cover + global_config = get_global_config_dict() + config = DeleteJsonlDatasetGitlabConfig.model_validate(global_config) + delete_jsonl_dataset_from_gitlab(config.dataset_name) + + +def upload_jsonl_dataset_to_hf_and_delete_gitlab_cli() -> None: # pragma: no cover + global_config = get_global_config_dict() + config = UploadJsonlDatasetHuggingFaceConfig.model_validate(global_config) + upload_jsonl_dataset_to_hf_maybe_delete(config, delete_from_gitlab=True) + + +def materialize_prompts_cli() -> None: # pragma: no cover + """CLI entry point for gym dataset render.""" + global_config_dict = get_global_config_dict( + global_config_dict_parser_config=GlobalConfigDictParserConfig( + initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + ) + config = MaterializePromptsConfig.model_validate(global_config_dict) + materialize_prompts(config.input_jsonl_fpath, config.prompt_config, config.output_jsonl_fpath) + + +def prepare_data(): # pragma: no cover + global_config_dict = get_global_config_dict( + global_config_dict_parser_config=GlobalConfigDictParserConfig( + initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + ) + + data_processor = TrainDataProcessor() + data_processor.run(global_config_dict) diff --git a/nemo_gym/cli/dev.py b/nemo_gym/cli/dev.py new file mode 100644 index 0000000000..f0f5ca41fc --- /dev/null +++ b/nemo_gym/cli/dev.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from subprocess import Popen + +from nemo_gym.config_types import BaseNeMoGymCLIConfig +from nemo_gym.global_config import get_global_config_dict + + +def dev_test(): # pragma: no cover + """ + Run core NeMo Gym tests with coverage reporting (runs pytest with --cov flag). + + Examples: + + ```bash + gym dev test + ``` + """ + global_config_dict = get_global_config_dict() + # Just here for help + BaseNeMoGymCLIConfig.model_validate(global_config_dict) + + proc = Popen("pytest --cov=. --durations=10", shell=True) + exit(proc.wait()) diff --git a/nemo_gym/cli.py b/nemo_gym/cli/env.py similarity index 81% rename from nemo_gym/cli.py rename to nemo_gym/cli/env.py index ec0ce66c0f..2e9f18a233 100644 --- a/nemo_gym/cli.py +++ b/nemo_gym/cli/env.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,16 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import asyncio import json import os -import platform import shlex -import sys -from copy import deepcopy from glob import glob -from importlib.metadata import entry_points -from importlib.metadata import version as md_version from os import makedirs from os.path import exists from pathlib import Path @@ -32,27 +28,26 @@ from time import sleep, time from typing import Dict, List, Optional, Tuple -import psutil import rich import uvicorn from devtools import pprint -from omegaconf import DictConfig, OmegaConf, open_dict +from omegaconf import DictConfig, OmegaConf from pydantic import Field from rich.table import Table from tqdm.auto import tqdm -from nemo_gym import PARENT_DIR, ROOT_DIR, __version__ -from nemo_gym.cli_setup_command import run_command, setup_env_command +from nemo_gym import PARENT_DIR, ROOT_DIR +from nemo_gym.cli.setup_command import run_command, setup_env_command from nemo_gym.config_types import BaseNeMoGymCLIConfig from nemo_gym.global_config import ( DRY_RUN_KEY_NAME, + JSON_OUTPUT_KEY_NAME, NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, NEMO_GYM_CONFIG_PATH_ENV_VAR_NAME, NEMO_GYM_RESERVED_TOP_LEVEL_KEYS, GlobalConfigDictParserConfig, get_global_config_dict, ) -from nemo_gym.rollout_collection import E2ERolloutCollectionConfig, RolloutCollectionConfig, RolloutCollectionHelper from nemo_gym.server_status import StatusCommand from nemo_gym.server_utils import ( HEAD_SERVER_KEY_NAME, @@ -62,7 +57,6 @@ ServerStatus, initialize_ray, ) -from nemo_gym.train_data_utils import TrainDataProcessor # Grace period after SIGINT before escalating to SIGKILL. Kept short so Ctrl-C is responsive. @@ -80,7 +74,7 @@ class RunConfig(BaseNeMoGymCLIConfig): ```bash config_paths="resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,\\ responses_api_models/openai_model/configs/openai_model.yaml" - ng_run "+config_paths=[${config_paths}]" + gym env start "+config_paths=[${config_paths}]" ``` """ @@ -96,7 +90,7 @@ class TestConfig(RunConfig): Examples: ```bash - ng_test +entrypoint=resources_servers/example_single_tool_call + gym env test +entrypoint=resources_servers/example_single_tool_call ``` """ @@ -416,7 +410,7 @@ def run( # Start servers with specific configs config_paths="resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,\\ responses_api_models/openai_model/configs/openai_model.yaml" - ng_run "+config_paths=[${config_paths}]" + gym env start "+config_paths=[${config_paths}]" ``` """ global_config_dict = get_global_config_dict(global_config_dict_parser_config=global_config_dict_parser_config) @@ -428,69 +422,6 @@ def run( rh.run_forever() -def e2e_rollout_collection(): # pragma: no cover - global_config_dict = get_global_config_dict() - - # Ensure we have the right config first thing - e2e_rollout_collection_config = E2ERolloutCollectionConfig.model_validate(global_config_dict) - - # Prepare data - data_processor_config_dict = deepcopy(global_config_dict) - with open_dict(data_processor_config_dict): - data_processor_config_dict["should_download"] = True - data_processor_config_dict["mode"] = "train_preparation" - - output_fpath = Path(e2e_rollout_collection_config.output_jsonl_fpath) - data_process_output_dir = output_fpath.parent / "preprocessed_datasets" - data_processor_config_dict["output_dirpath"] = str(data_process_output_dir) - - input_jsonl_fpath = data_process_output_dir / f"{e2e_rollout_collection_config.split}.jsonl" - should_skip_data_processing = ( - e2e_rollout_collection_config.reuse_existing_data_preparation and input_jsonl_fpath.exists() - ) - if not should_skip_data_processing: - if e2e_rollout_collection_config.reuse_existing_data_preparation: - print( - f"Even though the `reuse_existing_data_preparation=true` flag was set, we will still do data preparation since the final input jsonl fpath `{input_jsonl_fpath}` does not exist yet" - ) - - data_processor = TrainDataProcessor() - data_processor.run(data_processor_config_dict) - else: - print( - f"Skipping data preparation since `reuse_existing_data_preparation=true` and the final input jsonl fpath `{input_jsonl_fpath}` already exists" - ) - - # Convert to RolloutCollectionConfig - rollout_collection_config_dict = deepcopy(global_config_dict) - with open_dict(rollout_collection_config_dict): - assert input_jsonl_fpath.exists(), input_jsonl_fpath - rollout_collection_config_dict["input_jsonl_fpath"] = str(input_jsonl_fpath) - - rollout_collection_config = RolloutCollectionConfig.model_validate( - OmegaConf.to_container(rollout_collection_config_dict) - ) - - rh = RunHelper() - rh.start(None) - - rch = RolloutCollectionHelper() - - print( - f"""Output artifacts: -1. Preprocessed datasets: {data_processor_config_dict["output_dirpath"]} -2. Dataset file used for rollout collection: {rollout_collection_config_dict["input_jsonl_fpath"]} -3. Rollout collection results file: {output_fpath} -""" - ) - try: - asyncio.run(rch.run_from_config(rollout_collection_config)) - except KeyboardInterrupt: - pass - finally: - rh.shutdown() - - def _validate_data_single(test_config: TestConfig) -> None: # pragma: no cover if not test_config.should_validate_data: return @@ -515,7 +446,7 @@ def _validate_data_single(test_config: TestConfig) -> None: # pragma: no cover ), f"""You must run the example data validation for the example data found at {example_fpath}. Your command should look something like the following (you should update this command with your actual server config path): ```bash -ng_prepare_data "+config_paths=[{test_config._dir_path}/configs/{server_type_name}.yaml]" \\ +gym dataset collate "+config_paths=[{test_config._dir_path}/configs/{server_type_name}.yaml]" \\ +output_dirpath=data/{server_type_name} \\ +mode=example_validation ``` @@ -551,10 +482,10 @@ def _validate_data_single(test_config: TestConfig) -> None: # pragma: no cover # Server spinup example_multi_step_config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ resources_servers/example_multi_step/configs/example_multi_step.yaml" -ng_run "+config_paths=[${{example_multi_step_config_paths}}]" +gym env start "+config_paths=[${{example_multi_step_config_paths}}]" # Collect the rollouts -ng_collect_rollouts +agent_name=example_multi_step_simple_agent \ +gym eval run --no-serve +agent_name=example_multi_step_simple_agent \ +input_jsonl_fpath=resources_servers/example_multi_step/data/example.jsonl \ +output_jsonl_fpath=resources_servers/example_multi_step/data/example_rollouts.jsonl \ +limit=null @@ -606,7 +537,7 @@ class TestAllConfig(BaseNeMoGymCLIConfig): Examples: ```bash - ng_test_all + gym env test ``` """ @@ -694,7 +625,7 @@ def test_all(): # pragma: no cover case _: raise ValueError( f"""Hit unrecognized exit code {return_code} while running tests for {dir_path}. -You can rerun just these tests using `ng_test +entrypoint={dir_path}` or run detailed tests via `cd {dir_path} && source .venv/bin/activate && pytest`.""" +You can rerun just these tests using `gym env test +entrypoint={dir_path}` or run detailed tests via `cd {dir_path} && source .venv/bin/activate && pytest`.""" ) try: @@ -733,13 +664,13 @@ def test_all(): # pragma: no cover if tests_failed or tests_missing: print(f"""You can rerun just the server with failed or missing tests like: ```bash -ng_test +entrypoint={(tests_failed + tests_missing)[0]} +gym env test +entrypoint={(tests_failed + tests_missing)[0]} ``` """) if data_validation_failed: print(f"""You can rerun just the server with failed data validation like: ```bash -ng_test +entrypoint={data_validation_failed[0]} +should_validate_data=true +gym env test +entrypoint={data_validation_failed[0]} +should_validate_data=true ``` """) @@ -757,24 +688,6 @@ def test_all(): # pragma: no cover exit(1) -def dev_test(): # pragma: no cover - """ - Run core NeMo Gym tests with coverage reporting (runs pytest with --cov flag). - - Examples: - - ```bash - ng_dev_test - ``` - """ - global_config_dict = get_global_config_dict() - # Just here for help - BaseNeMoGymCLIConfig.model_validate(global_config_dict) - - proc = Popen("pytest --cov=. --durations=10", shell=True) - exit(proc.wait()) - - def init_resources_server(): # pragma: no cover """ Initialize a new resources server with template files and directory structure. @@ -782,7 +695,7 @@ def init_resources_server(): # pragma: no cover Examples: ```bash - ng_init_resources_server +entrypoint=resources_servers/my_server + gym env init +entrypoint=resources_servers/my_server ``` """ config_dict = get_global_config_dict() @@ -816,7 +729,7 @@ def init_resources_server(): # pragma: no cover verified: false # set true once the benchmark has been baselined and reviewed # Agent server config specifies the agent server to run and any additional components of the environment such as resources servers -{server_type_name}_simple_agent: # this agent instance's name — pass as +agent_name= to ng_collect_rollouts +{server_type_name}_simple_agent: # this agent instance's name — pass as --agent to gym eval run responses_api_agents: simple_agent: # built-in agent: runs the model with tool calls (up to max_steps); swap for your own agent dir entrypoint: app.py @@ -907,7 +820,7 @@ def dump_config(): # pragma: no cover Examples: ```bash - ng_dump_config "+config_paths=[,]" + gym env resolve "+config_paths=[,]" ``` """ global_config_dict = get_global_config_dict( @@ -922,39 +835,17 @@ def dump_config(): # pragma: no cover print(OmegaConf.to_yaml(global_config_dict, resolve=True)) -def display_help(): - """ - Display a list of available NeMo Gym CLI commands. - - Examples: - - ```bash - ng_help - ``` - """ - global_config_dict = get_global_config_dict() - # Just here for help - BaseNeMoGymCLIConfig.model_validate(global_config_dict) - - eps = entry_points().select(group="console_scripts") - project_scripts = {ep.name: ep.value for ep in eps if ep.name.startswith(("nemo_gym_", "ng_"))} - rich.print("""Run a command with `+h=true` or `+help=true` to see more detailed information! - -[bold]Available CLI scripts[/bold] ------------------""") - for script in project_scripts: - if not script.startswith("ng_"): - continue - - print(script) - - def status(): # pragma: no cover global_config_dict = get_global_config_dict() BaseNeMoGymCLIConfig.model_validate(global_config_dict) status_cmd = StatusCommand() servers = status_cmd.discover_servers() + + if global_config_dict.get(JSON_OUTPUT_KEY_NAME, False): + print(json.dumps([server.model_dump(mode="json") for server in servers])) + return + status_cmd.display_status(servers) @@ -980,7 +871,7 @@ def pip_list(): # pragma: no cover if not venv_path.exists(): print(f" Virtual environment not found at: {venv_path}") print(" Run tests or setup the server first using:") - print(f" ng_test +entrypoint={config.entrypoint}") + print(f" gym env test +entrypoint={config.entrypoint}") exit(1) pip_list_cmd = "uv pip list" @@ -1000,93 +891,3 @@ def pip_list(): # pragma: no cover proc = run_command(command, dir_path) return_code = proc.wait() exit(return_code) - - -class VersionConfig(BaseNeMoGymCLIConfig): - """ - Display gym version and system information. - - Examples: - - ```bash - # Display version information - ng_version - - # Output as JSON - ng_version +json=true - ``` - """ - - json_format: bool = Field(default=False, alias="json", description="Output in JSON format for programmatic use.") - - -def version(): # pragma: no cover - """Display gym version and system information.""" - global_config_dict = get_global_config_dict() - config = VersionConfig.model_validate(global_config_dict) - - json_output = config.json_format - - version_info = { - "nemo_gym": __version__, - "python": platform.python_version(), - "python_path": sys.executable, - "installation_path": str(PARENT_DIR), - } - - key_deps = [ - "openai", - "ray", - ] - - dependencies = {dep: md_version(dep) for dep in key_deps} - - version_info["dependencies"] = dependencies - - # System info - version_info["system"] = { - "os": f"{platform.system()} {platform.release()}", - "platform": platform.platform(), - "architecture": platform.machine(), - "processor": platform.processor() or "unknown", - "cpus": os.cpu_count(), - } - - # Memory info - mem = psutil.virtual_memory() - version_info["system"]["memory_gb"] = round(mem.total / (1024**3), 2) - - # Output - if json_output: - print(json.dumps(version_info)) - else: - output = f"""\ -NeMo Gym v{version_info["nemo_gym"]} -Python {version_info["python"]} ({version_info["python_path"]}) -Installation: {version_info["installation_path"]}""" - - if "dependencies" in version_info: - deps_lines = "\n".join(f" {dep}: {ver}" for dep, ver in version_info["dependencies"].items()) - sys_info = version_info["system"] - output += f""" - -Key Dependencies: -{deps_lines} - -System: - OS: {sys_info["os"]} - Platform: {sys_info["platform"]} - Architecture: {sys_info["architecture"]} - Processor: {sys_info["processor"]} - CPUs: {sys_info["cpus"]} - Memory: {sys_info["memory_gb"]} GB""" - - print(output) - - -def reinstall(): # pragma: no cover - global_config_dict = get_global_config_dict() - # Just here for help - BaseNeMoGymCLIConfig.model_validate(global_config_dict) - - Popen("uv sync --extra dev --group docs", shell=True).communicate() diff --git a/nemo_gym/cli/eval.py b/nemo_gym/cli/eval.py new file mode 100644 index 0000000000..8d3f5598df --- /dev/null +++ b/nemo_gym/cli/eval.py @@ -0,0 +1,439 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import difflib +import importlib +import json +from copy import deepcopy +from glob import glob +from multiprocessing import Pool +from pathlib import Path +from typing import Dict, List, Tuple + +import orjson +import rich +from omegaconf import DictConfig, OmegaConf, open_dict +from pydantic import Field +from rich.table import Table +from tqdm.auto import tqdm + +from nemo_gym.benchmarks import BENCHMARKS_DIR, BenchmarkConfig, _load_benchmarks_from_config_paths +from nemo_gym.cli.env import RunHelper +from nemo_gym.config_types import BaseNeMoGymCLIConfig, BenchmarkDatasetConfig +from nemo_gym.global_config import ( + JSON_OUTPUT_KEY_NAME, + POLICY_MODEL_KEY_NAME, + QUERY_KEY_NAME, + ROLLOUT_INDEX_KEY_NAME, + TASK_INDEX_KEY_NAME, + GlobalConfigDictParser, + GlobalConfigDictParserConfig, + get_first_server_config_dict, + get_global_config_dict, +) +from nemo_gym.reward_profile import RewardProfileConfig, RewardProfiler +from nemo_gym.rollout_collection import ( + E2ERolloutCollectionConfig, + RolloutAggregationConfig, + RolloutAggregationHelper, + RolloutCollectionConfig, + RolloutCollectionHelper, +) +from nemo_gym.train_data_utils import TrainDataProcessor + + +def _fuzzy_matches(query: str, *fields: str) -> bool: + """Whether `query` fuzzily matches any of `fields`: a substring or a close difflib match (token-aware).""" + needle = query.lower() + for field in fields: + if not field: + continue + haystack = field.lower() + if needle in haystack: + return True + tokens = haystack.replace("_", " ").replace("-", " ").split() + if difflib.get_close_matches(needle, [haystack, *tokens], n=1, cutoff=0.6): + return True + return False + + +def _benchmark_extras(bench: BenchmarkConfig) -> tuple[str, list[str]]: + """Resolve a benchmark's config to its `(domain, extra search terms)`. + + `BenchmarkConfig` flattens away the resources server name, the resources server `domain`, and the + dataset names. We re-resolve the config with the same parser `BenchmarkConfig` uses (so chained + `config_paths` / `_inherit_from` are applied) and read those fields back out for the domain column + and richer `gym search` matching. + """ + initial_config_dict = OmegaConf.load(bench.path) + if POLICY_MODEL_KEY_NAME not in initial_config_dict: + initial_config_dict = OmegaConf.merge( + initial_config_dict, GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT + ) + resolved = GlobalConfigDictParser().parse_no_environment(initial_global_config_dict=initial_config_dict) + + domain = "" + terms: list[str] = [] + for instance_name in resolved: + instance = resolved[instance_name] + if not isinstance(instance, (dict, DictConfig)): + continue + + resources_servers = instance.get("resources_servers") + if resources_servers: + terms.append(instance_name) # e.g. aime24_math_with_judge_resources_server + for rs_name, rs_config in resources_servers.items(): + terms.append(rs_name) # e.g. math_with_judge + found_domain = (rs_config or {}).get("domain") + if found_domain: + domain = str(found_domain) + + agents = instance.get("responses_api_agents") + if agents: + for agent_config in agents.values(): + for dataset in (agent_config or {}).get("datasets") or []: + if (dataset or {}).get("name"): + terms.append(dataset["name"]) + + if domain: + terms.append(domain) + return domain, terms + + +def list_benchmarks() -> None: + """CLI command: list available benchmarks, optionally filtered by a `query` (the `gym search` entry point).""" + global_config_dict = get_global_config_dict( + global_config_dict_parser_config=GlobalConfigDictParserConfig( + initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + ) + BaseNeMoGymCLIConfig.model_validate(global_config_dict) + + assert BENCHMARKS_DIR.exists(), "Missing benchmarks directory" + + config_paths = glob("**/config.yaml", root_dir=BENCHMARKS_DIR, recursive=True) + config_paths = [BENCHMARKS_DIR / p for p in config_paths] + config_paths = sorted(config_paths) + + benchmarks = _load_benchmarks_from_config_paths(config_paths) + + # Resolve the domain + richer search terms once per benchmark, for the domain column and `gym search`. + extras = {name: _benchmark_extras(bench) for name, bench in benchmarks.items()} + + # `gym search ` reuses this command, narrowing the listing to fuzzy matches across the + # benchmark name, agent name, resources server name, dataset names, and domain. + query = global_config_dict.get(QUERY_KEY_NAME) + if query: + benchmarks = { + name: bench + for name, bench in benchmarks.items() + if _fuzzy_matches(query, name, bench.agent_name, *extras[name][1]) + } + + if global_config_dict.get(JSON_OUTPUT_KEY_NAME, False): + payload = [ + { + "name": name, + "agent_name": bench.agent_name, + "domain": extras[name][0], + "num_repeats": bench.num_repeats, + } + for name, bench in benchmarks.items() + ] + print(json.dumps(payload)) + return + + if not benchmarks: + if query: + rich.print(f"[yellow]No benchmarks match '{query}'.[/yellow]") + return + rich.print("[yellow]No benchmarks found.[/yellow]") + rich.print(f"Expected benchmarks directory: {BENCHMARKS_DIR}") + return + + title = ( + f"Benchmarks matching '{query}' ({len(benchmarks)})" + if query + else f"Available benchmarks in NeMo Gym ({len(benchmarks)})" + ) + table = Table(title=title) + table.add_column("Benchmark name") + table.add_column("Domain") + table.add_column("Agent name") + table.add_column("Num repeats") + + for name, bench in benchmarks.items(): + table.add_row(name, extras[name][0], bench.agent_name, str(bench.num_repeats)) + + rich.print(table) + + +class PrepareBenchmarkConfig(BaseNeMoGymCLIConfig): + """ + Prepare benchmark data by running the benchmark's prepare.py script. + + The benchmark is identified from a config_paths entry pointing to a + benchmarks/*/config.yaml file. + + Examples: + + ```bash + gym eval prepare --benchmark aime24 + ``` + """ + + use_cached_prepared_benchmarks: bool = Field( + default=False, description="Skip benchmark preparation if the prepared file is already present" + ) + num_prepare_benchmark_processes: int = Field( + default=1, description="Number of processes to parallelize benchmark preparation" + ) + + +def _multiprocess_benchmark_prepare_fn(args): + benchmark_config: BenchmarkConfig + prepare_module_path: str + (benchmark_config, prepare_module_path) = args + + print(f"Preparing benchmark: {benchmark_config.name}") + + module = importlib.import_module(prepare_module_path) + output_fpath = module.prepare() + assert output_fpath.absolute() == benchmark_config.dataset.jsonl_fpath.absolute(), ( + f"Expected the actual prepared dataset output fpath to match the jsonl_fpath set in the config. Instead got {output_fpath=} jsonl_fpath={benchmark_config.dataset.jsonl_fpath}" + ) + print(f"Benchmark data prepared at: {output_fpath}") + + +def prepare_benchmark() -> None: + """CLI command: prepare benchmark data.""" + global_config_dict = get_global_config_dict( + global_config_dict_parser_config=GlobalConfigDictParserConfig( + initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, + ) + ) + prepare_benchmark_config = PrepareBenchmarkConfig.model_validate(global_config_dict) + + benchmarks_dict: Dict[str, BenchmarkConfig] = dict() + inspected_server_instances: List[str] = [] + for server_instance_name in global_config_dict: + server_config = global_config_dict[server_instance_name] + if not isinstance(server_config, (dict, DictConfig)) or "responses_api_agents" not in server_config: + continue + + inspected_server_instances.append(server_instance_name) + inner_server_config = get_first_server_config_dict(global_config_dict, server_instance_name) + + datasets: List[BenchmarkDatasetConfig] = [] + for dataset in inner_server_config.get("datasets") or []: + if dataset["type"] != "benchmark": + continue + + datasets.append(BenchmarkDatasetConfig.model_validate(dataset)) + + if len(datasets) < 1: + continue + + assert len(datasets) == 1, ( + f"Expected exactly 1 benchmark dataset for server instance `{server_instance_name}`, " + f"but found {len(datasets)}: {[d.name for d in datasets]}. " + "A benchmark config must define a single benchmark dataset." + ) + + dataset = datasets[0] + + benchmarks_dict[server_instance_name] = BenchmarkConfig( + name=dataset.name, + path=Path(""), + agent_name=server_instance_name, + num_repeats=dataset.num_repeats, + dataset=dataset, + ) + + assert benchmarks_dict, ( + "No benchmark config found. " + + ( + f"Inspected server instances {inspected_server_instances}, but none declared a `benchmark` dataset." + if inspected_server_instances + else "No server instances with `responses_api_agents` were found in the resolved config." + ) + + " Pass a benchmark with `gym eval prepare --benchmark ` (e.g. `--benchmark aime24`)." + ) + + # Validate all benchmarks before preparing any + prepare_script_missing: List[BenchmarkConfig] = [] + prepare_function_missing: List[BenchmarkConfig] = [] + + validated: List[Tuple[BenchmarkConfig, str]] = [] + already_prepared: List[BenchmarkConfig] = [] + for benchmark_config in benchmarks_dict.values(): + prepare_script_path = benchmark_config.dataset.prepare_script + if not prepare_script_path.exists(): + prepare_script_missing.append(benchmark_config) + continue + + prepare_module_path = ".".join(prepare_script_path.with_suffix("").parts) + module = importlib.import_module(prepare_module_path) + if not hasattr(module, "prepare"): + prepare_function_missing.append(benchmark_config) + continue + + is_already_prepared = benchmark_config.dataset.jsonl_fpath.exists() + if prepare_benchmark_config.use_cached_prepared_benchmarks and is_already_prepared: + already_prepared.append(benchmark_config) + continue + + validated.append((benchmark_config, prepare_module_path)) + + if already_prepared: + already_prepared_str = "".join(f"- {bc.name}: {bc.dataset.jsonl_fpath}\n" for bc in already_prepared) + already_prepared_str = f"""The following benchmarks have already been prepared. Since `use_cached_prepared_benchmarks=true`, we will skip re-preparation of those benchmarks. + {already_prepared_str}""" + print(already_prepared_str) + + errors_to_print = "" + if prepare_script_missing: + prepare_script_missing_str = "".join( + f"- {bc.name}: {bc.dataset.prepare_script}\n" for bc in prepare_script_missing + ) + errors_to_print += f"""The following benchmarks are missing a valid prepare script: +{prepare_script_missing_str} +""" + if prepare_function_missing: # pragma: no cover + prepare_function_missing_str = "".join( + f"- {bc.name}: {bc.dataset.prepare_script}\n" for bc in prepare_function_missing + ) + errors_to_print += f"""The following benchmarks have a prepare script, but are missing the prepare function: +{prepare_function_missing_str} +""" + if errors_to_print: + errors_to_print = f"""Did not prepare any benchmarks due to benchmark config errors. +{errors_to_print}""" + raise RuntimeError(errors_to_print) + + # Prepare after all validations pass + if prepare_benchmark_config.num_prepare_benchmark_processes > 1: # pragma: no cover + with Pool(processes=prepare_benchmark_config.num_prepare_benchmark_processes) as pool: + results = pool.imap_unordered(_multiprocess_benchmark_prepare_fn, validated) + list(tqdm(results, total=len(validated))) + else: + results = map(_multiprocess_benchmark_prepare_fn, validated) + list(tqdm(results, total=len(validated))) + + +def e2e_rollout_collection(): # pragma: no cover + global_config_dict = get_global_config_dict() + + # Ensure we have the right config first thing + e2e_rollout_collection_config = E2ERolloutCollectionConfig.model_validate(global_config_dict) + + # Prepare data + data_processor_config_dict = deepcopy(global_config_dict) + with open_dict(data_processor_config_dict): + data_processor_config_dict["should_download"] = True + data_processor_config_dict["mode"] = "train_preparation" + + output_fpath = Path(e2e_rollout_collection_config.output_jsonl_fpath) + data_process_output_dir = output_fpath.parent / "preprocessed_datasets" + data_processor_config_dict["output_dirpath"] = str(data_process_output_dir) + + input_jsonl_fpath = data_process_output_dir / f"{e2e_rollout_collection_config.split}.jsonl" + should_skip_data_processing = ( + e2e_rollout_collection_config.reuse_existing_data_preparation and input_jsonl_fpath.exists() + ) + if not should_skip_data_processing: + if e2e_rollout_collection_config.reuse_existing_data_preparation: + print( + f"Even though the `reuse_existing_data_preparation=true` flag was set, we will still do data preparation since the final input jsonl fpath `{input_jsonl_fpath}` does not exist yet" + ) + + data_processor = TrainDataProcessor() + data_processor.run(data_processor_config_dict) + else: + print( + f"Skipping data preparation since `reuse_existing_data_preparation=true` and the final input jsonl fpath `{input_jsonl_fpath}` already exists" + ) + + # Convert to RolloutCollectionConfig + rollout_collection_config_dict = deepcopy(global_config_dict) + with open_dict(rollout_collection_config_dict): + assert input_jsonl_fpath.exists(), input_jsonl_fpath + rollout_collection_config_dict["input_jsonl_fpath"] = str(input_jsonl_fpath) + + rollout_collection_config = RolloutCollectionConfig.model_validate( + OmegaConf.to_container(rollout_collection_config_dict) + ) + + rh = RunHelper() + rh.start(None) + + rch = RolloutCollectionHelper() + + print( + f"""Output artifacts: +1. Preprocessed datasets: {data_processor_config_dict["output_dirpath"]} +2. Dataset file used for rollout collection: {rollout_collection_config_dict["input_jsonl_fpath"]} +3. Rollout collection results file: {output_fpath} +""" + ) + try: + asyncio.run(rch.run_from_config(rollout_collection_config)) + except KeyboardInterrupt: + pass + finally: + rh.shutdown() + + +def collect_rollouts(): # pragma: no cover + config = RolloutCollectionConfig.model_validate(get_global_config_dict()) + rch = RolloutCollectionHelper() + + asyncio.run(rch.run_from_config(config)) + + +def aggregate_rollouts(): # pragma: no cover + config = RolloutAggregationConfig.model_validate(get_global_config_dict()) + rah = RolloutAggregationHelper() + + asyncio.run(rah.run_from_config(config)) + + +def reward_profile(): # pragma: no cover + config = RewardProfileConfig.model_validate(get_global_config_dict()) + + with open(config.materialized_inputs_jsonl_fpath) as f: + rows = list(map(orjson.loads, f)) + + with open(config.rollouts_jsonl_fpath) as f: + results = list(map(orjson.loads, f)) + + # Results may be out of order. + results.sort(key=lambda r: (r[TASK_INDEX_KEY_NAME], r[ROLLOUT_INDEX_KEY_NAME])) + + rp = RewardProfiler() + group_level_metrics, agent_level_metrics = rp.profile_from_data( + rows, results, allow_partial_rollouts=config.allow_partial_rollouts + ) + completion_summary = rp.profile_completion_summary(rows, results) + reward_profiling_fpath, agent_level_metrics_fpath = rp.write_to_disk( + group_level_metrics, agent_level_metrics, Path(config.rollouts_jsonl_fpath) + ) + + print(f"""Profiling outputs: +Reward profile completion: {completion_summary["completed_rollout_rows"]}/{completion_summary["expected_rollout_rows"]} rollout rows ({completion_summary["reward_profile_completion_pct"]:.2f}%) +Input rows: {completion_summary["total_input_rows"]} total; {completion_summary["complete_input_rows"]} complete; {completion_summary["partial_input_rows"]} partial; {completion_summary["missing_input_rows"]} without rollouts dropped from output. +Reward profiling outputs: {reward_profiling_fpath} +Agent-level metrics: {agent_level_metrics_fpath}""") diff --git a/nemo_gym/cli/general.py b/nemo_gym/cli/general.py new file mode 100644 index 0000000000..615a5bab75 --- /dev/null +++ b/nemo_gym/cli/general.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import platform +import sys +from importlib.metadata import version as md_version +from subprocess import Popen + +import psutil +from pydantic import Field + +from nemo_gym import PARENT_DIR, __version__ +from nemo_gym.config_types import BaseNeMoGymCLIConfig +from nemo_gym.global_config import JSON_OUTPUT_KEY_NAME, get_global_config_dict + + +class VersionConfig(BaseNeMoGymCLIConfig): + """ + Display gym version and system information. + + Examples: + + ```bash + # Display version information + gym --version + + # Output as JSON + gym --version --json + ``` + """ + + json_format: bool = Field(default=False, alias="json", description="Output in JSON format for programmatic use.") + + +def version(): # pragma: no cover + """Display gym version and system information.""" + global_config_dict = get_global_config_dict() + # Just here for help. + VersionConfig.model_validate(global_config_dict) + + version_info = { + "nemo_gym": __version__, + "python": platform.python_version(), + "python_path": sys.executable, + "installation_path": str(PARENT_DIR), + } + + key_deps = [ + "openai", + "ray", + ] + + dependencies = {dep: md_version(dep) for dep in key_deps} + + version_info["dependencies"] = dependencies + + # System info + version_info["system"] = { + "os": f"{platform.system()} {platform.release()}", + "platform": platform.platform(), + "architecture": platform.machine(), + "processor": platform.processor() or "unknown", + "cpus": os.cpu_count(), + } + + # Memory info + mem = psutil.virtual_memory() + version_info["system"]["memory_gb"] = round(mem.total / (1024**3), 2) + + if global_config_dict.get(JSON_OUTPUT_KEY_NAME, False): + print(json.dumps(version_info)) + else: + output = f"""\ +NeMo Gym v{version_info["nemo_gym"]} +Python {version_info["python"]} ({version_info["python_path"]}) +Installation: {version_info["installation_path"]}""" + + if "dependencies" in version_info: + deps_lines = "\n".join(f" {dep}: {ver}" for dep, ver in version_info["dependencies"].items()) + sys_info = version_info["system"] + output += f""" + +Key Dependencies: +{deps_lines} + +System: + OS: {sys_info["os"]} + Platform: {sys_info["platform"]} + Architecture: {sys_info["architecture"]} + Processor: {sys_info["processor"]} + CPUs: {sys_info["cpus"]} + Memory: {sys_info["memory_gb"]} GB""" + + print(output) + + +def reinstall(): # pragma: no cover + global_config_dict = get_global_config_dict() + # Just here for help + BaseNeMoGymCLIConfig.model_validate(global_config_dict) + + Popen("uv sync --extra dev --group docs", shell=True).communicate() diff --git a/nemo_gym/cli/legacy.py b/nemo_gym/cli/legacy.py new file mode 100644 index 0000000000..59c5dab631 --- /dev/null +++ b/nemo_gym/cli/legacy.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Backward-compatibility shim for the legacy ``ng_*`` / ``nemo_gym_*`` commands. + +Every legacy console script points here; the script name (``sys.argv[0]``) identifies which +command was invoked. We print a one-time deprecation notice mapping it to the new ``gym`` +command, then re-enter the ``gym`` router so the command keeps working (REQ 7). +""" + +import sys +from pathlib import Path + +from nemo_gym.cli.main import dispatch +from nemo_gym.cli.main import main as gym_main + + +# Legacy command (ng_/nemo_gym_ prefix stripped) -> equivalent `gym` subcommand tokens. +LEGACY = { + "run": ["env", "start"], + "test": ["env", "test"], + "test_all": ["env", "test"], + "dev_test": ["dev", "test"], + "init_resources_server": ["env", "init"], + "list_benchmarks": ["list", "benchmarks"], + "prepare_benchmark": ["eval", "prepare"], + "collect_rollouts": ["eval", "run", "--no-serve"], + "e2e_collect_rollouts": ["eval", "run"], + "aggregate_rollouts": ["eval", "aggregate"], + "materialize_prompts": ["dataset", "render"], + "reward_profile": ["eval", "profile"], + "upload_dataset_to_gitlab": ["dataset", "upload", "--storage", "gitlab"], + "download_dataset_from_gitlab": ["dataset", "download", "--storage", "gitlab"], + "prepare_data": ["dataset", "collate"], + "upload_dataset_to_hf": ["dataset", "upload"], + "download_dataset_from_hf": ["dataset", "download"], + "gitlab_to_hf_dataset": ["dataset", "migrate"], + "delete_dataset_from_gitlab": ["dataset", "rm"], + "dump_config": ["env", "resolve"], + "help": ["--help"], + "status": ["env", "status"], + "pip_list": ["env", "packages"], + "version": ["--version"], +} + + +def main() -> None: + alias = Path(sys.argv[0]).name + key = alias.removeprefix("nemo_gym_").removeprefix("ng_") + + # `reinstall` has no `gym` equivalent (`gym install` was dropped); point users at the uv command it runs. + if key == "reinstall": + print( + f"⚠ `{alias}` is deprecated and will be removed in a future release; " + f"run `uv sync --extra dev --group docs` instead.", + file=sys.stderr, + ) + dispatch("nemo_gym.cli.general:reinstall", sys.argv[1:]) + return + + tokens = LEGACY.get(key) + if tokens is None: + # Reached only if a new `ng_*` / `nemo_gym_*` console script is wired to `legacy:main` + # without a matching `LEGACY` entry (a packaging bug, caught by tests), or a user invokes + # an alias that no longer exists. Fail loudly with a non-zero exit code. + print( + f"⚠ `{alias}` has no known `gym` equivalent. Run `gym --help` to see available commands.", + file=sys.stderr, + ) + sys.exit(1) + + print( + f"⚠ `{alias}` is deprecated and will be removed in a future release; use `gym {' '.join(tokens)}` instead.", + file=sys.stderr, + ) + # Re-enter the gym router with the equivalent subcommand, preserving the user's Hydra overrides. + sys.argv = [sys.argv[0], *tokens, *sys.argv[1:]] + gym_main() diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py new file mode 100644 index 0000000000..7f99b17701 --- /dev/null +++ b/nemo_gym/cli/main.py @@ -0,0 +1,571 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import difflib +import importlib +import re +import sys +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field +from pathlib import Path + +from nemo_gym import WORKING_DIR + + +VERSION_TARGET = "nemo_gym.cli.general:version" + + +def _did_you_mean(value: str, candidates: Iterable[str]) -> str: + """A ` Did you mean \\`X\\`?` fragment for the closest candidate to `value`, or `""` if none is close enough.""" + matches = difflib.get_close_matches(value, list(candidates), n=1) + return f" Did you mean `{matches[0]}`?" if matches else "" + + +class _GymArgumentParser(argparse.ArgumentParser): + """ArgumentParser that appends a difflib "did you mean?" hint to invalid-choice errors. + + Covers mistyped commands/groups and bad --flag choices (e.g. --storage), since argparse validates all of them + as choices against the registry baked into the parser. + """ + + def error(self, message: str) -> None: + match = re.search(r"invalid choice: '([^']+)' \(choose from (.+)\)", message) + if match: + typo, choices = match.group(1), re.findall(r"'([^']+)'", match.group(2)) + message += _did_you_mean(typo, choices) + super().error(message) + + +@dataclass(frozen=True) +class Flag: + # Register this flag's argument(s) on a command's subparser. + register: Callable[[argparse.ArgumentParser], None] + # Turn the parsed value into leading Hydra override tokens (default: contributes nothing). + translate_to_hydra: Callable[[argparse.Namespace], list[str]] = lambda args: [] + + +@dataclass(frozen=True) +class Command: + # What to run: either a "module:function" string (lazily imported and called with no args), + # or a callable(args, overrides) that owns dispatch (e.g. picks the target from parsed flags). + target: str | Callable[[argparse.Namespace, list[str]], None] + # One-line help shown in the parent listing and atop this command's own --help. + summary: str | None = None + # Flags this command accepts; reusable ones (e.g. CONFIG) are shared across commands. + flags: tuple[Flag, ...] = field(default_factory=tuple) + + +def dispatch(target: str, overrides: list[str]) -> None: + module_path, func_name = target.split(":") + # Drop the parsed command tokens so the downstream Hydra parsing only sees overrides. + sys.argv = [sys.argv[0], *overrides] + func = getattr(importlib.import_module(module_path), func_name) + func() + + +def _value_flag( + name: str, hydra_key: str, flag_help: str, *, aliases: tuple[str, ...] = (), choices: tuple[str, ...] | None = None +) -> Flag: + """A `--name VALUE` flag that maps to the Hydra override `+=VALUE` (omitted when unset).""" + dest = name.replace("-", "_") + return Flag( + register=lambda p: p.add_argument(f"--{name}", *aliases, dest=dest, choices=choices, help=flag_help), + translate_to_hydra=lambda args: ( + [f"+{hydra_key}={getattr(args, dest)}"] if getattr(args, dest) is not None else [] + ), + ) + + +def _bool_flag(name: str, hydra_key: str, flag_help: str) -> Flag: + """A `--name` store_true flag that maps to the Hydra override `+=true` when set.""" + dest = name.replace("-", "_") + return Flag( + register=lambda p: p.add_argument(f"--{name}", action="store_true", help=flag_help), + translate_to_hydra=lambda args: [f"+{hydra_key}=true"] if getattr(args, dest) else [], + ) + + +# Shared flag: load Gym config files. Reused by every command that reads server/benchmark configs. +CONFIG = Flag( + register=lambda p: p.add_argument( + "--config", + action="append", + metavar="PATH", + help="Config file to load; repeatable. Maps to +config_paths=[...].", + ), + translate_to_hydra=lambda args: [f"+config_paths=[{','.join(args.config)}]"] if args.config else [], +) + +# Shared flag: select the storage backend. Reused by `dataset upload` and `dataset download`. +STORAGE = Flag( + register=lambda p: p.add_argument( + "--storage", choices=("hf", "gitlab"), default="hf", help="Storage backend (default: hf)." + ) +) + +# Shared model-server flags. Reused by commands that spin up / target a model server (`eval run`, `env start`). +# --model is the served model identifier across all backends: an API model name, an HF id, or a local checkpoint +# path, interpreted per --model-type (e.g. a path/HF id to serve with local_vllm_model). +MODEL = _value_flag( + "model", + "policy_model_name", + "Model name, HF id, or local checkpoint path (interpreted per --model-type).", + aliases=("-m",), +) +MODEL_URL = _value_flag("model-url", "policy_base_url", "Model server base URL.") +MODEL_API_KEY = _value_flag("model-api-key", "policy_api_key", "Model server API key.") + +# Shared flag: select a single resources server by name. Reused by `env test`, `env init`, and `env packages`. +RESOURCES_SERVER = Flag( + register=lambda p: p.add_argument("--resources-server", metavar="NAME", help="Name of the resources server."), + translate_to_hydra=lambda args: ( + [f"+entrypoint=resources_servers/{args.resources_server}"] if args.resources_server else [] + ), +) + +# Shared flag: emit machine-readable JSON instead of human output. Reused by reporting commands (version, list, +# env status). Each command reads the reserved `json` config key ad hoc via +# global_config_dict.get(JSON_OUTPUT_KEY_NAME) (see general.py, eval.py, env.py). +JSON = _bool_flag("json", "json", "Output as JSON for programmatic use.") + +# Positional search query for `gym search`; surfaced to the listing command as the `query` config key. +QUERY = Flag( + register=lambda p: p.add_argument("query", metavar="QUERY", help="Substring to match against component names."), + translate_to_hydra=lambda args: [f"+query={args.query}"] if getattr(args, "query", None) else [], +) + + +# Asset selector flag -> (parent dir, configs subdir, default config flavor). All accept `name` or `name/flavor`, +# resolving to `//[/].yaml`. A None default flavor falls back to the server name. +_ASSETS = { + "benchmark": ("benchmarks", "", "config"), + "environment": ("environments", "", "config"), + "resources-server": ("resources_servers", "configs", None), + "model-type": ("responses_api_models", "configs", None), +} + + +def _asset_config_path(flag: str, value: str, search_dirs: tuple[str, ...] = ()) -> str: + """Map a named asset (`name` or `name/flavor`) to its config path. + + Searches WORKING_DIR (built-ins) first, then any user-registered --search-dir roots. + """ + parent, subdir, default_flavor = _ASSETS[flag] + server_name, _, config_flavor = value.partition("/") + config_flavor = config_flavor or default_flavor or server_name + config_dir = f"{parent}/{server_name}/{subdir}".rstrip("/") + path = f"{config_dir}/{config_flavor}.yaml" + + # Match in WORKING_DIR (built-ins) and every --search-dir root; dedupe roots that resolve to the same file. + roots = [WORKING_DIR, *(Path(d) for d in search_dirs)] + matches: list[Path] = [] + + for root in roots: + candidate = root / path + if candidate.exists(): + matches.append(candidate.resolve()) + + if len(matches) > 1: + matches_str = ", ".join(f"`{m}`" for m in matches) + raise ValueError( + f"`--{flag} {value}` is ambiguous: it matches multiple configs ({matches_str}). " + f"Pass the intended config directly with `--config ` instead." + ) + if matches: + return str(matches[0]) + + # No match: suggest the closest real name across all roots (a config flavor when the server exists, else a + # server name) and report the full paths that were searched. + available = ", ".join(set(f"`{(root / config_dir).resolve()}`" for root in roots if (root / config_dir).is_dir())) + typo = config_flavor + candidates = [p.stem for root in roots for p in (root / config_dir).glob("*.yaml")] + + if len(candidates) == 0: + available = ", ".join(set(f"`{(root / parent).resolve()}`" for root in roots if (root / parent).is_dir())) + typo = server_name + candidates = [ + child.name + for root in roots + if (root / parent).is_dir() + for child in (root / parent).iterdir() + if child.is_dir() + ] + + raise ValueError( + f"`--{flag} {value}` was specified which implies config `{path}`, which does not exist.{_did_you_mean(typo, candidates)} " + f"See available {flag} configs in {available}." + ) + + +def _asset_selector(flag: str) -> Flag: + """A `-- NAME` selector that resolves the named asset to a config and adds it to +config_paths.""" + dest = flag.replace("-", "_") + return Flag( + register=lambda p: p.add_argument(f"--{flag}", metavar="NAME", help=f"Load the named {flag} config."), + translate_to_hydra=lambda args: ( + [ + f"+config_paths=[{_asset_config_path(flag, getattr(args, dest), tuple(getattr(args, 'search_dir', None) or ()))}]" + ] + if getattr(args, dest) + else [] + ), + ) + + +BENCHMARK = _asset_selector("benchmark") +ENVIRONMENT = _asset_selector("environment") +RESOURCES_SERVER_CONFIG = _asset_selector("resources-server") +MODEL_TYPE = _asset_selector("model-type") + +# Shared flag: register extra root dirs to search for named components. Consumed by the asset selectors above +# (not emitted as a Hydra override). Reused by every command that accepts a -- NAME selector. +SEARCH_DIR = Flag( + register=lambda p: p.add_argument( + "--search-dir", + action="append", + metavar="DIR", + help="Extra root directory to search for named components; repeatable.", + ), +) + + +def _merge_config_paths(overrides: list[str]) -> list[str]: + """Coalesce all `+config_paths=[...]` tokens (from --config and asset selectors) into one (Hydra rejects dupes).""" + prefix = "+config_paths=[" + paths: list[str] = [] + rest: list[str] = [] + for token in overrides: + if token.startswith(prefix) and token.endswith("]"): + paths.extend(p for p in token[len(prefix) : -1].split(",") if p) + else: + rest.append(token) + return ([f"+config_paths=[{','.join(paths)}]"] if paths else []) + rest + + +def _eval_run(args: argparse.Namespace, overrides: list[str]) -> None: + target = "nemo_gym.cli.eval:collect_rollouts" if args.no_serve else "nemo_gym.cli.eval:e2e_rollout_collection" + dispatch(target, overrides) + + +def _env_test(args: argparse.Namespace, overrides: list[str]) -> None: + # Run a single server's tests if +entrypoint was passed. No need to check for + # --resources-server because it is translated to +entrypoint in the flag definition. + + has_entrypoint = any(override.lstrip("+").split("=", 1)[0] == "entrypoint" for override in overrides) + dispatch("nemo_gym.cli.env:test" if has_entrypoint else "nemo_gym.cli.env:test_all", overrides) + + +def _dataset_upload(args: argparse.Namespace, overrides: list[str]) -> None: + targets = { + "hf": "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli", + "gitlab": "nemo_gym.cli.dataset:upload_jsonl_dataset_cli", + } + dispatch(targets[args.storage], overrides) + + +def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: + targets = { + "hf": "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli", + "gitlab": "nemo_gym.cli.dataset:download_jsonl_dataset_cli", + } + dispatch(targets[args.storage], overrides) + + +# One-line help for each command group, shown in `gym --help`. +GROUPS = { + "list": "List available components. As of now, only benchmarks are available.", + "dataset": "Manage datasets.", + "env": "Develop and run environments.", + "eval": "Run evaluations.", + "dev": "Contributor helpers.", +} + +COMMANDS = { + "list benchmarks": Command( + target="nemo_gym.cli.eval:list_benchmarks", summary="List available benchmarks.", flags=(JSON,) + ), + "search": Command( + target="nemo_gym.cli.eval:list_benchmarks", + summary="Search available components (currently benchmarks) by name; like `list` filtered to a query.", + flags=(QUERY, JSON), + ), + "dataset upload": Command( + target=_dataset_upload, + summary="Upload a prepared dataset to HF (default) or GitLab.", + flags=( + STORAGE, + _value_flag("input", "input_jsonl_fpath", "Local JSONL file to upload.", aliases=("-i",)), + _value_flag("name", "dataset_name", "Dataset name."), + # GitLab stores it as `version`, HF as `revision`; emit both and let each backend keep its own. + Flag( + register=lambda p: p.add_argument( + "--revision", dest="revision", help="Dataset revision (version) to upload." + ), + translate_to_hydra=lambda args: ( + # we set both version and revision because GitLab and HF use different keys + # and we extra="ignore" so it's safe to set both + [f"+version={args.revision}", f"+revision={args.revision}"] if args.revision is not None else [] + ), + ), + _value_flag("split", "split", "Dataset split (HF only)."), + _bool_flag("create-pr", "create_pr", "Open a pull request instead of committing directly (HF only)."), + ), + ), + "dataset download": Command( + target=_dataset_download, + summary="Download a dataset from HF (default) or GitLab.", + flags=( + STORAGE, + _value_flag("repo-id", "repo_id", "HF repo id, e.g. org/dataset (HF only)."), + _value_flag("name", "dataset_name", "Dataset name (GitLab only)."), + # NOTE(martas): HF download does not allow to specify revision + _value_flag("revision", "version", "Dataset version (GitLab only)."), + _value_flag( + "artifact", "artifact_fpath", "Remote file to fetch (GitLab: required; HF: optional raw file)." + ), + _value_flag("output", "output_fpath", "Local destination file.", aliases=("-o",)), + _value_flag( + "output-dir", "output_dirpath", "Local destination directory; needed for all splits (HF only)." + ), + _value_flag("split", "split", "Dataset split (HF only)."), + ), + ), + "dataset rm": Command( + target="nemo_gym.cli.dataset:delete_jsonl_dataset_from_gitlab_cli", + summary="Delete a dataset from GitLab.", + flags=(_value_flag("name", "dataset_name", "Name of the dataset to delete."),), + ), + "dataset migrate": Command( + target="nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli", + summary="Transfer a dataset from GitLab to HF.", + flags=( + _value_flag("input", "input_jsonl_fpath", "Local JSONL file to upload to HF.", aliases=("-i",)), + _value_flag("name", "dataset_name", "Dataset name."), + _value_flag("revision", "revision", "Dataset revision (HF)."), + _value_flag("split", "split", "Dataset split."), + _bool_flag("create-pr", "create_pr", "Open a pull request instead of committing directly."), + ), + ), + "dataset render": Command( + target="nemo_gym.cli.dataset:materialize_prompts_cli", + summary="Generate a dataset preview.", + flags=( + _value_flag("input", "input_jsonl_fpath", "Raw input JSONL file.", aliases=("-i",)), + _value_flag("prompt-config", "prompt_config", "Prompt template YAML to apply."), + _value_flag("output", "output_jsonl_fpath", "Output JSONL file.", aliases=("-o",)), + ), + ), + "dataset collate": Command( + target="nemo_gym.cli.dataset:prepare_data", + summary="Validate and collate the dataset.", + flags=( + CONFIG, + RESOURCES_SERVER_CONFIG, + SEARCH_DIR, + _value_flag("mode", "mode", "Data preparation mode.", choices=("train_preparation", "example_validation")), + _value_flag("output-dir", "output_dirpath", "Output directory for the prepared data."), + _bool_flag("download", "should_download", "Download source datasets before collating."), + ), + ), + "env init": Command( + target="nemo_gym.cli.env:init_resources_server", + summary="Scaffold config for a new server, benchmark, or agent.", + flags=(RESOURCES_SERVER,), + ), + "env resolve": Command( + target="nemo_gym.cli.env:dump_config", + summary="Resolve the final config from configs, flags, and overrides.", + flags=(CONFIG,), + ), + "env packages": Command( + target="nemo_gym.cli.env:pip_list", + summary="Print pip packages for the selected resources server.", + flags=( + RESOURCES_SERVER, + _bool_flag("outdated", "outdated", "List only outdated packages."), + Flag( + register=lambda p: p.add_argument( + "--json", action="store_true", help="Output the package list as JSON." + ), + translate_to_hydra=lambda args: ["+format=json"] if args.json else [], + ), + ), + ), + "env test": Command( + target=_env_test, + summary="Test the resources server(s); runs all if no resources server is given.", + flags=(RESOURCES_SERVER,), + ), + "env start": Command( + target="nemo_gym.cli.env:run", + summary="Start the servers.", + flags=( + CONFIG, + BENCHMARK, + ENVIRONMENT, + RESOURCES_SERVER_CONFIG, + MODEL_TYPE, + SEARCH_DIR, + MODEL, + MODEL_URL, + MODEL_API_KEY, + ), + ), + "env status": Command(target="nemo_gym.cli.env:status", summary="Print the server status.", flags=(JSON,)), + "eval prepare": Command( + target="nemo_gym.cli.eval:prepare_benchmark", + summary="Prepare benchmark data and dump it to disk.", + flags=(CONFIG, BENCHMARK, SEARCH_DIR), + ), + "eval run": Command( + target=_eval_run, + summary="Collate data, start servers, and collect rollouts.", + flags=( + CONFIG, + BENCHMARK, + ENVIRONMENT, + RESOURCES_SERVER_CONFIG, + MODEL_TYPE, + SEARCH_DIR, + Flag( + register=lambda p: p.add_argument( + "--no-serve", + action="store_true", + help="Collect against already-running servers instead of starting them.", + ) + ), + _bool_flag("resume", "resume_from_cache", "Resume from cached rollouts instead of recollecting."), + _value_flag("agent", "agent_name", "Agent to collect rollouts with.", aliases=("-a",)), + _value_flag("input", "input_jsonl_fpath", "Input tasks JSONL file.", aliases=("-i",)), + _value_flag("output", "output_jsonl_fpath", "Output rollouts JSONL file.", aliases=("-o",)), + _value_flag("limit", "limit", "Maximum number of tasks to run."), + _value_flag("num-repeats", "num_repeats", "Number of rollouts per task."), + _value_flag("prompt-config", "prompt_config", "Prompt template YAML to apply."), + _value_flag("concurrency", "num_samples_in_parallel", "Maximum number of concurrent samples."), + _value_flag("split", "split", "Dataset split to use (train, validation, or benchmark)."), + MODEL, + MODEL_URL, + MODEL_API_KEY, + _value_flag("temperature", "responses_create_params.temperature", "Sampling temperature."), + _value_flag("top-p", "responses_create_params.top_p", "Nucleus sampling top-p."), + _value_flag("max-output-tokens", "responses_create_params.max_output_tokens", "Maximum output tokens."), + ), + ), + "eval aggregate": Command( + target="nemo_gym.cli.eval:aggregate_rollouts", + summary="Aggregate sharded rollout results.", + flags=( + CONFIG, + _value_flag( + "input-glob", + "input_glob", + "Glob (or comma-separated globs) matching the rollout shards to aggregate.", + aliases=("-i",), + ), + _value_flag( + "output", + "output_jsonl_fpath", + "Path for the merged rollouts and aggregate-metrics file.", + aliases=("-o",), + ), + ), + ), + "eval profile": Command( + target="nemo_gym.cli.eval:reward_profile", + summary="Compute a reward profile from rollouts.", + flags=( + _value_flag( + "inputs", "materialized_inputs_jsonl_fpath", "Materialized inputs JSONL fed to rollout collection." + ), + _value_flag("rollouts", "rollouts_jsonl_fpath", "Rollouts JSONL produced by collection."), + ), + ), + "dev test": Command(target="nemo_gym.cli.dev:dev_test", summary="Run NeMo Gym's unit tests."), +} + + +def _add_leaf(subparsers: argparse._SubParsersAction, name: str, command: Command) -> None: + leaf = subparsers.add_parser(name, help=command.summary, description=command.summary) + # `_parser=leaf` so error reporting (and flag "did you mean?" hints) uses this command's own options/prog. + leaf.set_defaults(_command=command, _parser=leaf) + leaf.add_argument("-v", "--verbose", action="store_true", help="Set logging level to DEBUG.") + for flag in command.flags: + flag.register(leaf) + + +def build_parser() -> argparse.ArgumentParser: + # _GymArgumentParser propagates to every subparser (argparse defaults parser_class to type(self)). + parser = _GymArgumentParser(prog="gym", add_help=True) + parser.add_argument("--version", action="store_true", help="Show the NeMo Gym version and exit.") + parser.add_argument("--json", action="store_true", help="With --version, output as JSON.") + parser.set_defaults(_parser=parser) + + subparsers = parser.add_subparsers() + groups: dict[str, argparse._SubParsersAction] = {} + + for command_name, command in COMMANDS.items(): + parts = command_name.split() + if len(parts) == 1: + _add_leaf(subparsers, parts[0], command) + continue + + group_name, action_name = parts + if group_name not in groups: + group_parser = subparsers.add_parser( + group_name, help=GROUPS.get(group_name), description=GROUPS.get(group_name) + ) + group_parser.set_defaults(_parser=group_parser) + groups[group_name] = group_parser.add_subparsers() + _add_leaf(groups[group_name], action_name, command) + + return parser + + +def main() -> None: + parser = build_parser() + args, overrides = parser.parse_known_args() + + # Hydra overrides never start with "-" so we treat them as unknown flags. + unknown_flags = [token for token in overrides if token.startswith("-")] + if unknown_flags: + error_parser = getattr(args, "_parser", parser) + known_options = [opt for action in error_parser._actions for opt in action.option_strings] + hints = "".join(_did_you_mean(flag.split("=", 1)[0], known_options) for flag in unknown_flags) + error_parser.error(f"unrecognized arguments: {' '.join(unknown_flags)}{hints}") + + if args.version: + dispatch(VERSION_TARGET, ["+json=true", *overrides] if args.json else overrides) + return + + command = getattr(args, "_command", None) + if command is None: + args._parser.print_help() + sys.exit(1) + + try: + translated = [token for flag in command.flags for token in flag.translate_to_hydra(args)] + except ValueError as exc: + sys.stderr.write(f"{getattr(args, '_parser', parser).prog}: error: {exc}\n") + sys.exit(2) + + # --config and the asset selectors all emit +config_paths; coalesce them into one token. + overrides = _merge_config_paths(translated + overrides) + # --verbose flows through the config (as +verbose=true) so it reaches spun-up servers, not just this process. + if getattr(args, "verbose", False): + overrides = ["+verbose=true", *overrides] + if callable(command.target): + command.target(args, overrides) + else: + dispatch(command.target, overrides) diff --git a/nemo_gym/cli/setup_command.py b/nemo_gym/cli/setup_command.py new file mode 100644 index 0000000000..c1940d4933 --- /dev/null +++ b/nemo_gym/cli/setup_command.py @@ -0,0 +1,204 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import importlib.metadata +import os +from os import environ +from pathlib import Path +from subprocess import Popen +from sys import stderr, stdout + +from omegaconf import DictConfig + +from nemo_gym import PARENT_DIR +from nemo_gym.global_config import ( + HEAD_SERVER_DEPS_KEY_NAME, + NEMO_GYM_LOG_DIR_KEY_NAME, + PIP_INSTALL_VERBOSE_KEY_NAME, + PYTHON_VERSION_KEY_NAME, + SKIP_VENV_IF_PRESENT_KEY_NAME, + UV_CACHE_DIR_KEY_NAME, + UV_PIP_SET_PYTHON_KEY_NAME, + UV_VENV_DIR_KEY_NAME, + get_global_config_dict, +) + + +def _get_nemo_gym_install_flags() -> str: + """ + Build uv pip install flags for nemo-gym in sub-venvs. + + Supports: + - Pre-release versions via NEMO_GYM_ALLOW_PRERELEASE=true + - Custom PyPI indexes via UV_INDEX_URL, UV_EXTRA_INDEX_URL, UV_INDEX_STRATEGY + - Auto-detection of parent venv version for consistency + + Returns: + String of flags to add to 'uv pip install nemo-gym' + Example: "--pre --index-url https://test.pypi.org/simple/ ==0.2.1rc0" + """ + flags = "" + + # 1. Pre-release flag + allow_prerelease = os.getenv("NEMO_GYM_ALLOW_PRERELEASE", "").lower() == "true" + if allow_prerelease: + flags += "--pre " + # When pre-releases are enabled, also use unsafe-best-match strategy if not already set + if not os.getenv("UV_INDEX_STRATEGY"): + flags += "--index-strategy unsafe-best-match " + # Pin fastapi<1.0 to avoid broken test.pypi package + flags += "'fastapi<1.0' " + + # 2. Index URLs (respects uv's standard env vars) + index_url = os.getenv("UV_INDEX_URL") + if index_url: + flags += f"--index-url {index_url} " + + extra_index_url = os.getenv("UV_EXTRA_INDEX_URL") + if extra_index_url: + flags += f"--extra-index-url {extra_index_url} " + + # Explicit index strategy (overrides auto-set above) + index_strategy = os.getenv("UV_INDEX_STRATEGY") + if index_strategy: + flags += f"--index-strategy {index_strategy} " + + return flags + + +def _get_nemo_gym_version_spec(is_editable_install: bool) -> str: + """ + Detect nemo-gym version from parent venv and return version specifier. + + Args: + is_editable_install: Whether nemo-gym is installed in editable mode in parent venv + + Returns: + Version specifier string (e.g., "==0.2.1rc0") or empty string + """ + # Don't pin version for editable installs (development mode) + if is_editable_install: + return "" + + try: + parent_version = importlib.metadata.version("nemo-gym") + # Pin to exact version for consistency between parent and sub-venvs + return f"=={parent_version}" + except importlib.metadata.PackageNotFoundError: + # nemo-gym not installed in parent venv (shouldn't happen, but be safe) + return "" + + +def setup_env_command(dir_path: Path, global_config_dict: DictConfig, prefix: str) -> str: + head_server_deps = global_config_dict[HEAD_SERVER_DEPS_KEY_NAME] + + root_venv_path = global_config_dict[UV_VENV_DIR_KEY_NAME] + if Path(root_venv_path).resolve() != PARENT_DIR.resolve(): + venv_path = Path(root_venv_path, *dir_path.parts[-2:], ".venv").absolute() + else: + venv_path = (dir_path / ".venv").absolute() + + uv_venv_cmd = f"uv venv --seed --allow-existing --python {global_config_dict[PYTHON_VERSION_KEY_NAME]} {venv_path}" + + venv_python_fpath = venv_path / "bin/python" + venv_activate_fpath = venv_path / "bin/activate" + skip_venv_if_present = global_config_dict[SKIP_VENV_IF_PRESENT_KEY_NAME] + should_skip_venv_setup = bool(skip_venv_if_present) and venv_python_fpath.exists() and venv_activate_fpath.exists() + + # explicitly set python path if specified. In Google colab, gym env start fails due to uv pip install falls back to system python (/usr) without this and errors. + # not needed for most clusters. should be safe in all scenarios, but only minimally tested outside of colab. + # see discussion and examples here: https://github.com/NVIDIA-NeMo/Gym/pull/526#issuecomment-3676230383 + uv_pip_set_python = global_config_dict.get(UV_PIP_SET_PYTHON_KEY_NAME, False) + uv_pip_python_flag = f"--python {venv_python_fpath} " if uv_pip_set_python else "" + + verbose_flag = "-v " if global_config_dict.get(PIP_INSTALL_VERBOSE_KEY_NAME) else "" + + is_editable_install = (dir_path.resolve() / "../../pyproject.toml").exists() + + if should_skip_venv_setup: + env_setup_cmd = f"source {venv_activate_fpath}" + else: + has_pyproject_toml = (dir_path / "pyproject.toml").exists() + has_requirements_txt = (dir_path / "requirements.txt").exists() + if has_pyproject_toml and has_requirements_txt: + raise RuntimeError( + f"Found both pyproject.toml and requirements.txt for uv venv setup in server dir: {dir_path}. Please only use one or the other!" + ) + elif has_pyproject_toml: + if is_editable_install: + install_cmd = ( + f"""uv pip install {verbose_flag}{uv_pip_python_flag}'-e .' {" ".join(head_server_deps)}""" + ) + else: + # install nemo-gym from pypi instead of relative path in pyproject.toml + # with support for pre-releases, custom indexes, and version pinning + install_flags = _get_nemo_gym_install_flags() + version_spec = _get_nemo_gym_version_spec(is_editable_install) + install_cmd = ( + f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}nemo-gym{version_spec} && """ + f"""uv pip install {verbose_flag}{uv_pip_python_flag}--no-sources '-e .' {" ".join(head_server_deps)}""" + ) + elif has_requirements_txt: + if is_editable_install: + install_cmd = f"""uv pip install {verbose_flag}{uv_pip_python_flag}-r requirements.txt {" ".join(head_server_deps)}""" + else: + # install nemo-gym from pypi instead of relative path in requirements.txt + # with support for pre-releases, custom indexes, and version pinning + install_flags = _get_nemo_gym_install_flags() + version_spec = _get_nemo_gym_version_spec(is_editable_install) + install_cmd = ( + f"""(echo 'nemo-gym{version_spec}' && grep -v -F '../..' requirements.txt) | """ + f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}-r /dev/stdin {" ".join(head_server_deps)}""" + ) + else: + raise RuntimeError( + f"Missing pyproject.toml or requirements.txt for uv venv setup in server dir: {dir_path}" + ) + + prefix_cmd = f" > >(sed 's/^/({prefix}) /') 2> >(sed 's/^/({prefix}) /' >&2)" + env_setup_cmd = f"{uv_venv_cmd}{prefix_cmd} && source {venv_activate_fpath} && {install_cmd}{prefix_cmd}" + + return f"cd {dir_path} && {env_setup_cmd}" + + +def run_command(command: str, working_dir_path: Path, server_name: str = "") -> Popen: + global_config_dict = get_global_config_dict() + + work_dir = f"{working_dir_path.absolute()}" + custom_env = environ.copy() + py_path = custom_env.get("PYTHONPATH", None) + if py_path is not None: + custom_env["PYTHONPATH"] = f"{work_dir}:{py_path}" + else: + custom_env["PYTHONPATH"] = work_dir + + custom_env["UV_CACHE_DIR"] = global_config_dict[UV_CACHE_DIR_KEY_NAME] + + log_dir = global_config_dict.get(NEMO_GYM_LOG_DIR_KEY_NAME) + if log_dir: + safe_name = (server_name or working_dir_path.name).replace("/", "_") + log_path = Path(log_dir) / f"{safe_name}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + command = f"set -o pipefail; ({command}) 2>&1 | tee -a {log_path}" + + redirect_stdout = stdout + redirect_stderr = stderr + return Popen( + command, + executable="/bin/bash", + shell=True, + env=custom_env, + stdout=redirect_stdout, + stderr=redirect_stderr, + ) diff --git a/nemo_gym/cli_setup_command.py b/nemo_gym/cli_setup_command.py index 9e5a72181d..9d112ab251 100644 --- a/nemo_gym/cli_setup_command.py +++ b/nemo_gym/cli_setup_command.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,193 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import importlib.metadata -import os -from os import environ -from pathlib import Path -from subprocess import Popen -from sys import stderr, stdout +"""Backward-compatibility shim: this module moved to `nemo_gym.cli.setup_command`. -from omegaconf import DictConfig - -from nemo_gym import PARENT_DIR -from nemo_gym.global_config import ( - HEAD_SERVER_DEPS_KEY_NAME, - NEMO_GYM_LOG_DIR_KEY_NAME, - PIP_INSTALL_VERBOSE_KEY_NAME, - PYTHON_VERSION_KEY_NAME, - SKIP_VENV_IF_PRESENT_KEY_NAME, - UV_CACHE_DIR_KEY_NAME, - UV_PIP_SET_PYTHON_KEY_NAME, - UV_VENV_DIR_KEY_NAME, - get_global_config_dict, -) - - -def _get_nemo_gym_install_flags() -> str: - """ - Build uv pip install flags for nemo-gym in sub-venvs. - - Supports: - - Pre-release versions via NEMO_GYM_ALLOW_PRERELEASE=true - - Custom PyPI indexes via UV_INDEX_URL, UV_EXTRA_INDEX_URL, UV_INDEX_STRATEGY - - Auto-detection of parent venv version for consistency - - Returns: - String of flags to add to 'uv pip install nemo-gym' - Example: "--pre --index-url https://test.pypi.org/simple/ ==0.2.1rc0" - """ - flags = "" - - # 1. Pre-release flag - allow_prerelease = os.getenv("NEMO_GYM_ALLOW_PRERELEASE", "").lower() == "true" - if allow_prerelease: - flags += "--pre " - # When pre-releases are enabled, also use unsafe-best-match strategy if not already set - if not os.getenv("UV_INDEX_STRATEGY"): - flags += "--index-strategy unsafe-best-match " - # Pin fastapi<1.0 to avoid broken test.pypi package - flags += "'fastapi<1.0' " - - # 2. Index URLs (respects uv's standard env vars) - index_url = os.getenv("UV_INDEX_URL") - if index_url: - flags += f"--index-url {index_url} " - - extra_index_url = os.getenv("UV_EXTRA_INDEX_URL") - if extra_index_url: - flags += f"--extra-index-url {extra_index_url} " - - # Explicit index strategy (overrides auto-set above) - index_strategy = os.getenv("UV_INDEX_STRATEGY") - if index_strategy: - flags += f"--index-strategy {index_strategy} " - - return flags - - -def _get_nemo_gym_version_spec(is_editable_install: bool) -> str: - """ - Detect nemo-gym version from parent venv and return version specifier. - - Args: - is_editable_install: Whether nemo-gym is installed in editable mode in parent venv - - Returns: - Version specifier string (e.g., "==0.2.1rc0") or empty string - """ - # Don't pin version for editable installs (development mode) - if is_editable_install: - return "" - - try: - parent_version = importlib.metadata.version("nemo-gym") - # Pin to exact version for consistency between parent and sub-venvs - return f"=={parent_version}" - except importlib.metadata.PackageNotFoundError: - # nemo-gym not installed in parent venv (shouldn't happen, but be safe) - return "" +Accessing its public helpers here resolves them lazily from the new location with a +`DeprecationWarning`. Remove once the old import path is dropped. +""" +from nemo_gym.cli._compat import moved_attr_getter -def setup_env_command(dir_path: Path, global_config_dict: DictConfig, prefix: str) -> str: - head_server_deps = global_config_dict[HEAD_SERVER_DEPS_KEY_NAME] - root_venv_path = global_config_dict[UV_VENV_DIR_KEY_NAME] - if Path(root_venv_path).resolve() != PARENT_DIR.resolve(): - venv_path = Path(root_venv_path, *dir_path.parts[-2:], ".venv").absolute() - else: - venv_path = (dir_path / ".venv").absolute() - - uv_venv_cmd = f"uv venv --seed --allow-existing --python {global_config_dict[PYTHON_VERSION_KEY_NAME]} {venv_path}" - - venv_python_fpath = venv_path / "bin/python" - venv_activate_fpath = venv_path / "bin/activate" - skip_venv_if_present = global_config_dict[SKIP_VENV_IF_PRESENT_KEY_NAME] - should_skip_venv_setup = bool(skip_venv_if_present) and venv_python_fpath.exists() and venv_activate_fpath.exists() - - # explicitly set python path if specified. In Google colab, ng_run fails due to uv pip install falls back to system python (/usr) without this and errors. - # not needed for most clusters. should be safe in all scenarios, but only minimally tested outside of colab. - # see discussion and examples here: https://github.com/NVIDIA-NeMo/Gym/pull/526#issuecomment-3676230383 - uv_pip_set_python = global_config_dict.get(UV_PIP_SET_PYTHON_KEY_NAME, False) - uv_pip_python_flag = f"--python {venv_python_fpath} " if uv_pip_set_python else "" - - verbose_flag = "-v " if global_config_dict.get(PIP_INSTALL_VERBOSE_KEY_NAME) else "" - - is_editable_install = (dir_path.resolve() / "../../pyproject.toml").exists() - - if should_skip_venv_setup: - env_setup_cmd = f"source {venv_activate_fpath}" - else: - has_pyproject_toml = (dir_path / "pyproject.toml").exists() - has_requirements_txt = (dir_path / "requirements.txt").exists() - if has_pyproject_toml and has_requirements_txt: - raise RuntimeError( - f"Found both pyproject.toml and requirements.txt for uv venv setup in server dir: {dir_path}. Please only use one or the other!" - ) - elif has_pyproject_toml: - if is_editable_install: - install_cmd = ( - f"""uv pip install {verbose_flag}{uv_pip_python_flag}'-e .' {" ".join(head_server_deps)}""" - ) - else: - # install nemo-gym from pypi instead of relative path in pyproject.toml - # with support for pre-releases, custom indexes, and version pinning - install_flags = _get_nemo_gym_install_flags() - version_spec = _get_nemo_gym_version_spec(is_editable_install) - install_cmd = ( - f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}nemo-gym{version_spec} && """ - f"""uv pip install {verbose_flag}{uv_pip_python_flag}--no-sources '-e .' {" ".join(head_server_deps)}""" - ) - elif has_requirements_txt: - if is_editable_install: - install_cmd = f"""uv pip install {verbose_flag}{uv_pip_python_flag}-r requirements.txt {" ".join(head_server_deps)}""" - else: - # install nemo-gym from pypi instead of relative path in requirements.txt - # with support for pre-releases, custom indexes, and version pinning - install_flags = _get_nemo_gym_install_flags() - version_spec = _get_nemo_gym_version_spec(is_editable_install) - install_cmd = ( - f"""(echo 'nemo-gym{version_spec}' && grep -v -F '../..' requirements.txt) | """ - f"""uv pip install {verbose_flag}{uv_pip_python_flag}{install_flags}-r /dev/stdin {" ".join(head_server_deps)}""" - ) - else: - raise RuntimeError( - f"Missing pyproject.toml or requirements.txt for uv venv setup in server dir: {dir_path}" - ) - - prefix_cmd = f" > >(sed 's/^/({prefix}) /') 2> >(sed 's/^/({prefix}) /' >&2)" - env_setup_cmd = f"{uv_venv_cmd}{prefix_cmd} && source {venv_activate_fpath} && {install_cmd}{prefix_cmd}" - - return f"cd {dir_path} && {env_setup_cmd}" - - -def run_command(command: str, working_dir_path: Path, server_name: str = "") -> Popen: - global_config_dict = get_global_config_dict() - - work_dir = f"{working_dir_path.absolute()}" - custom_env = environ.copy() - py_path = custom_env.get("PYTHONPATH", None) - if py_path is not None: - custom_env["PYTHONPATH"] = f"{work_dir}:{py_path}" - else: - custom_env["PYTHONPATH"] = work_dir - - custom_env["UV_CACHE_DIR"] = global_config_dict[UV_CACHE_DIR_KEY_NAME] - - log_dir = global_config_dict.get(NEMO_GYM_LOG_DIR_KEY_NAME) - if log_dir: - safe_name = (server_name or working_dir_path.name).replace("/", "_") - log_path = Path(log_dir) / f"{safe_name}.log" - log_path.parent.mkdir(parents=True, exist_ok=True) - command = f"set -o pipefail; ({command}) 2>&1 | tee -a {log_path}" - - redirect_stdout = stdout - redirect_stderr = stderr - return Popen( - command, - executable="/bin/bash", - shell=True, - env=custom_env, - stdout=redirect_stdout, - stderr=redirect_stderr, - ) +__getattr__ = moved_attr_getter( + __name__, + { + "run_command": "nemo_gym.cli.setup_command", + "setup_env_command": "nemo_gym.cli.setup_command", + }, +) diff --git a/nemo_gym/config_types.py b/nemo_gym/config_types.py index c80121ec43..75e9b59f54 100644 --- a/nemo_gym/config_types.py +++ b/nemo_gym/config_types.py @@ -156,7 +156,7 @@ class UploadJsonlDatasetGitlabConfig(BaseNeMoGymCLIConfig): Examples: ```bash - ng_upload_dataset_to_gitlab \ + gym dataset upload --storage gitlab \ +dataset_name=example_multi_step \ +version=0.0.1 \ +input_jsonl_fpath=data/train.jsonl @@ -181,7 +181,7 @@ class DownloadJsonlDatasetGitlabConfig(JsonlDatasetGitlabIdentifer, BaseNeMoGymC Examples: ```bash - ng_download_dataset_from_gitlab \ + gym dataset download --storage gitlab \ +dataset_name=example_multi_step \ +version=0.0.1 \ +artifact_fpath=train.jsonl \ @@ -202,7 +202,7 @@ class DeleteJsonlDatasetGitlabConfig(BaseNeMoGymCLIConfig): Examples: ```bash - ng_delete_dataset_from_gitlab +dataset_name=old_dataset + gym dataset rm +dataset_name=old_dataset ``` """ @@ -225,7 +225,7 @@ class BaseUploadJsonlDatasetHuggingFaceConfig(BaseNeMoGymCLIConfig): ```bash resource_config_path="resources_servers/example_multi_step/configs/example_multi_step.yaml" - ng_upload_dataset_to_hf \ + gym dataset upload \ +dataset_name=my_dataset \ +input_jsonl_fpath=data/train.jsonl \ +resource_config_path=${resource_config_path} @@ -272,13 +272,13 @@ class UploadJsonlDatasetHuggingFaceConfig(BaseUploadJsonlDatasetHuggingFaceConfi Upload a JSONL dataset to HuggingFace Hub and automatically delete from GitLab after successful upload. This command always deletes the dataset from GitLab after uploading to HuggingFace. - Use `ng_upload_dataset_to_hf` if you want optional deletion control. + Use `gym dataset upload` if you want optional deletion control. Examples: ```bash resource_config_path="resources_servers/example_multi_step/configs/example_multi_step.yaml" - ng_gitlab_to_hf_dataset \ + gym dataset migrate \ +dataset_name=my_dataset \ +input_jsonl_fpath=data/train.jsonl \ +resource_config_path=${resource_config_path} @@ -304,7 +304,7 @@ class UploadJsonlDatasetHuggingFaceMaybeDeleteConfig(BaseUploadJsonlDatasetHuggi ```bash resource_config_path="resources_servers/example_multi_step/configs/example_multi_step.yaml" - ng_upload_dataset_to_hf \ + gym dataset upload \ +dataset_name=my_dataset \ +input_jsonl_fpath=data/train.jsonl \ +resource_config_path=${resource_config_path} \ @@ -324,7 +324,7 @@ class DownloadJsonlDatasetHuggingFaceConfig(JsonlDatasetHuggingFaceIdentifer, Ba Examples: ```bash - ng_download_dataset_from_hf \ + gym dataset download \ +repo_id=NVIDIA/NeMo-Gym-Math-example_multi_step-v1 \ +artifact_fpath=train.jsonl \ +output_fpath=data/train.jsonl diff --git a/nemo_gym/dataset_orchestrator.py b/nemo_gym/dataset_orchestrator.py index 80401564d7..baf6686c44 100644 --- a/nemo_gym/dataset_orchestrator.py +++ b/nemo_gym/dataset_orchestrator.py @@ -15,15 +15,11 @@ from typing import Union from nemo_gym.config_types import ( - DeleteJsonlDatasetGitlabConfig, - DownloadJsonlDatasetHuggingFaceConfig, UploadJsonlDatasetHuggingFaceConfig, UploadJsonlDatasetHuggingFaceMaybeDeleteConfig, ) from nemo_gym.gitlab_utils import delete_model_from_gitlab, is_model_in_gitlab -from nemo_gym.hf_utils import download_hf_dataset_as_jsonl from nemo_gym.hf_utils import upload_jsonl_dataset as upload_jsonl_dataset_to_hf -from nemo_gym.server_utils import get_global_config_dict def delete_jsonl_dataset_from_gitlab(gitlab_model_name: str) -> None: # pragma: no cover @@ -59,31 +55,17 @@ def upload_jsonl_dataset_to_hf_maybe_delete( delete_jsonl_dataset_from_gitlab(gitlab_model_name) -def upload_jsonl_dataset_to_hf_cli() -> None: # pragma: no cover - global_config = get_global_config_dict() - config = UploadJsonlDatasetHuggingFaceMaybeDeleteConfig.model_validate(global_config) - upload_jsonl_dataset_to_hf_maybe_delete(config, delete_from_gitlab=config.delete_from_gitlab) +# Backward-compatibility shims (CLI refactor): these CLI entry points moved to `nemo_gym.cli.dataset`. +# Re-exported lazily to avoid a circular import; accessing them emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 -def upload_jsonl_dataset_to_hf_and_delete_gitlab_cli() -> None: # pragma: no cover - global_config = get_global_config_dict() - config = UploadJsonlDatasetHuggingFaceConfig.model_validate(global_config) - upload_jsonl_dataset_to_hf_maybe_delete(config, delete_from_gitlab=True) - - -def download_jsonl_dataset_from_hf_cli() -> None: # pragma: no cover - global_config = get_global_config_dict() - config = DownloadJsonlDatasetHuggingFaceConfig.model_validate(global_config) - - if config.artifact_fpath: - print(f"Downloading file '{config.artifact_fpath}' from '{config.repo_id}'...") - else: - print(f"Downloading '{config.split or 'all'}' split(s) from '{config.repo_id}'...") - - download_hf_dataset_as_jsonl(config) - - -def delete_jsonl_dataset_from_gitlab_cli() -> None: # pragma: no cover - global_config = get_global_config_dict() - config = DeleteJsonlDatasetGitlabConfig.model_validate(global_config) - delete_jsonl_dataset_from_gitlab(config.dataset_name) +__getattr__ = moved_attr_getter( + __name__, + { + "upload_jsonl_dataset_to_hf_cli": "nemo_gym.cli.dataset", + "upload_jsonl_dataset_to_hf_and_delete_gitlab_cli": "nemo_gym.cli.dataset", + "download_jsonl_dataset_from_hf_cli": "nemo_gym.cli.dataset", + "delete_jsonl_dataset_from_gitlab_cli": "nemo_gym.cli.dataset", + }, +) diff --git a/nemo_gym/gitlab_utils.py b/nemo_gym/gitlab_utils.py index 23c7a2898f..4404a4e59c 100644 --- a/nemo_gym/gitlab_utils.py +++ b/nemo_gym/gitlab_utils.py @@ -66,7 +66,7 @@ def upload_jsonl_dataset( filename = Path(config.input_jsonl_fpath).name DownloadJsonlDatasetGitlabConfig print(f"""Download this artifact: -ng_download_dataset_from_gitlab \\ +gym dataset download --storage gitlab \\ +dataset_name={config.dataset_name} \\ +version={config.version} \\ +artifact_fpath={filename} \\ @@ -74,12 +74,6 @@ def upload_jsonl_dataset( """) -def upload_jsonl_dataset_cli() -> None: # pragma: no cover - global_config = get_global_config_dict() - config = UploadJsonlDatasetGitlabConfig.model_validate(global_config) - upload_jsonl_dataset(config) - - def download_jsonl_dataset( config: DownloadJsonlDatasetGitlabConfig, ) -> None: # pragma: no cover @@ -100,12 +94,6 @@ def download_jsonl_dataset( f.write(response.content.decode()) -def download_jsonl_dataset_cli() -> None: # pragma: no cover - global_config = get_global_config_dict() - config = DownloadJsonlDatasetGitlabConfig.model_validate(global_config) - download_jsonl_dataset(config) - - def is_model_in_gitlab(model_name: str) -> bool: # pragma: no cover client = create_mlflow_client() @@ -122,3 +110,17 @@ def is_model_in_gitlab(model_name: str) -> bool: # pragma: no cover def delete_model_from_gitlab(model_name: str) -> None: # pragma: no cover client = create_mlflow_client() client.delete_registered_model(model_name) + + +# Backward-compatibility shims (CLI refactor): these CLI entry points moved to `nemo_gym.cli.dataset`. +# Re-exported lazily to avoid a circular import; accessing them emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 + + +__getattr__ = moved_attr_getter( + __name__, + { + "upload_jsonl_dataset_cli": "nemo_gym.cli.dataset", + "download_jsonl_dataset_cli": "nemo_gym.cli.dataset", + }, +) diff --git a/nemo_gym/global_config.py b/nemo_gym/global_config.py index 4773a15b2b..c3282afd08 100644 --- a/nemo_gym/global_config.py +++ b/nemo_gym/global_config.py @@ -12,6 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging +import sys from argparse import ArgumentParser from collections import defaultdict from copy import deepcopy @@ -76,6 +78,9 @@ # real config value (e.g. a literal "???" or a DictConfig) for "missing". _MISSING_REF = object() NEMO_GYM_LOG_DIR_KEY_NAME = "nemo_gym_log_dir" +VERBOSE_KEY_NAME = "verbose" +JSON_OUTPUT_KEY_NAME = "json" +QUERY_KEY_NAME = "query" NEMO_GYM_RESERVED_TOP_LEVEL_KEYS = [ CONFIG_PATHS_KEY_NAME, ENTRYPOINT_KEY_NAME, @@ -98,6 +103,9 @@ INHERIT_FROM_KEY_NAME, COPY_KEY_NAME, NEMO_GYM_LOG_DIR_KEY_NAME, + VERBOSE_KEY_NAME, + JSON_OUTPUT_KEY_NAME, + QUERY_KEY_NAME, ] # Data keys @@ -191,6 +199,12 @@ def inner_hydra_wrapper(cfg: DictConfig) -> DictConfig: inner_hydra_wrapper() + # Hydra installs a console log handler on stdout; move it to stderr so command stdout stays machine-readable + # (e.g. `gym ... --json`). Diagnostics belong on stderr; only the requested data goes to stdout. + for handler in logging.getLogger().handlers: + if isinstance(handler, logging.StreamHandler) and getattr(handler, "stream", None) is sys.stdout: + handler.setStream(sys.stderr) + global_config_dict: DictConfig = config_list[0] return global_config_dict @@ -724,6 +738,7 @@ def get_global_config_dict( _GLOBAL_CONFIG_DICT = global_config_dict + _apply_verbosity(global_config_dict) return global_config_dict set_global_config_dict( @@ -731,9 +746,18 @@ def get_global_config_dict( global_config_dict_parser_cls=global_config_dict_parser_cls, ) + _apply_verbosity(_GLOBAL_CONFIG_DICT) return _GLOBAL_CONFIG_DICT +def _apply_verbosity(global_config_dict: DictConfig) -> None: + """Set logging to DEBUG when `verbose` is in the config. Runs in the CLI process and, because the + config dict is forwarded to every spun-up server, in each server process too.""" + if global_config_dict.get(VERBOSE_KEY_NAME): + logging.basicConfig(level=logging.DEBUG) + logging.getLogger().setLevel(logging.DEBUG) + + def set_global_config_dict( global_config_dict_parser_config: Optional[GlobalConfigDictParserConfig] = None, global_config_dict_parser_cls: Type[GlobalConfigDictParser] = GlobalConfigDictParser, diff --git a/nemo_gym/prompt.py b/nemo_gym/prompt.py index 8a4d10f5bc..ed07d80eb0 100644 --- a/nemo_gym/prompt.py +++ b/nemo_gym/prompt.py @@ -29,7 +29,6 @@ from nemo_gym import PARENT_DIR from nemo_gym.config_types import BaseNeMoGymCLIConfig -from nemo_gym.global_config import GlobalConfigDictParserConfig, get_global_config_dict class PromptConfig(BaseModel): @@ -152,7 +151,7 @@ class MaterializePromptsConfig(BaseNeMoGymCLIConfig): Examples: ```bash - ng_materialize_prompts \\ + gym dataset render \\ +input_jsonl_fpath=data/my_dataset.jsonl \\ +prompt_config=/path/to/my_prompt.yaml \\ +output_jsonl_fpath=my_dataset_materialized.jsonl @@ -164,12 +163,14 @@ class MaterializePromptsConfig(BaseNeMoGymCLIConfig): output_jsonl_fpath: str = Field(description="Output path for materialized JSONL with populated prompts.") -def materialize_prompts_cli() -> None: # pragma: no cover - """CLI entry point for ng_materialize_prompts.""" - global_config_dict = get_global_config_dict( - global_config_dict_parser_config=GlobalConfigDictParserConfig( - initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, - ) - ) - config = MaterializePromptsConfig.model_validate(global_config_dict) - materialize_prompts(config.input_jsonl_fpath, config.prompt_config, config.output_jsonl_fpath) +# Backward-compatibility shim (CLI refactor): this CLI entry point moved to `nemo_gym.cli.dataset`. +# Re-exported lazily to avoid a circular import; accessing it emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 + + +__getattr__ = moved_attr_getter( + __name__, + { + "materialize_prompts_cli": "nemo_gym.cli.dataset", + }, +) diff --git a/nemo_gym/reward_profile.py b/nemo_gym/reward_profile.py index e7bc8eb6d6..1de3d354f0 100644 --- a/nemo_gym/reward_profile.py +++ b/nemo_gym/reward_profile.py @@ -31,15 +31,14 @@ AGENT_REF_KEY_NAME, ROLLOUT_INDEX_KEY_NAME, TASK_INDEX_KEY_NAME, - get_global_config_dict, ) class RewardProfileConfig(BaseNeMoGymCLIConfig): materialized_inputs_jsonl_fpath: str = Field( - description="The file path of the materialized inputs as output by ng_collect_rollouts." + description="The file path of the materialized inputs as output by `gym eval run`." ) - rollouts_jsonl_fpath: str = Field(description="The file path of the rollouts as output by ng_collect_rollouts.") + rollouts_jsonl_fpath: str = Field(description="The file path of the rollouts as output by `gym eval run`.") allow_partial_rollouts: bool = Field( default=False, description="Allow reward profiling from partial rollout outputs by dropping input rows with no completed rollouts.", @@ -703,29 +702,14 @@ def compute_aggregate_metrics( ) -def reward_profile(): # pragma: no cover - config = RewardProfileConfig.model_validate(get_global_config_dict()) +# Backward-compatibility shim (CLI refactor): this CLI entry point moved to `nemo_gym.cli.eval`. +# Re-exported lazily to avoid a circular import; accessing it emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 - with open(config.materialized_inputs_jsonl_fpath) as f: - rows = list(map(orjson.loads, f)) - with open(config.rollouts_jsonl_fpath) as f: - results = list(map(orjson.loads, f)) - - # Results may be out of order. - results.sort(key=lambda r: (r[TASK_INDEX_KEY_NAME], r[ROLLOUT_INDEX_KEY_NAME])) - - rp = RewardProfiler() - group_level_metrics, agent_level_metrics = rp.profile_from_data( - rows, results, allow_partial_rollouts=config.allow_partial_rollouts - ) - completion_summary = rp.profile_completion_summary(rows, results) - reward_profiling_fpath, agent_level_metrics_fpath = rp.write_to_disk( - group_level_metrics, agent_level_metrics, Path(config.rollouts_jsonl_fpath) - ) - - print(f"""Profiling outputs: -Reward profile completion: {completion_summary["completed_rollout_rows"]}/{completion_summary["expected_rollout_rows"]} rollout rows ({completion_summary["reward_profile_completion_pct"]:.2f}%) -Input rows: {completion_summary["total_input_rows"]} total; {completion_summary["complete_input_rows"]} complete; {completion_summary["partial_input_rows"]} partial; {completion_summary["missing_input_rows"]} without rollouts dropped from output. -Reward profiling outputs: {reward_profiling_fpath} -Agent-level metrics: {agent_level_metrics_fpath}""") +__getattr__ = moved_attr_getter( + __name__, + { + "reward_profile": "nemo_gym.cli.eval", + }, +) diff --git a/nemo_gym/rollout_collection.py b/nemo_gym/rollout_collection.py index 86ae07d1ff..1aa7d3c08c 100644 --- a/nemo_gym/rollout_collection.py +++ b/nemo_gym/rollout_collection.py @@ -44,7 +44,6 @@ from nemo_gym.server_utils import ( GlobalAIOHTTPAsyncClientConfig, ServerClient, - get_global_config_dict, get_response_json, is_global_aiohttp_client_request_debug_enabled, is_global_aiohttp_client_setup, @@ -128,7 +127,7 @@ class SharedRolloutCollectionConfig(BaseNeMoGymCLIConfig): description=( "Skip the post-rollout aggregate-metrics computation and file write. " "Used when sharding rollouts across multiple jobs that will be aggregated together " - "afterward by `ng_aggregate_rollouts`." + "afterward by `gym eval aggregate`." ), ) @@ -140,7 +139,7 @@ class E2ERolloutCollectionConfig(SharedRolloutCollectionConfig): Examples: ```bash - ng_collect_rollouts \ + gym eval run \ +output_jsonl_fpath=weather_rollouts.jsonl \ +num_samples_in_parallel=10 ``` @@ -157,7 +156,7 @@ class RolloutCollectionConfig(SharedRolloutCollectionConfig): Examples: ```bash - ng_collect_rollouts \ + gym eval run --no-serve \ +agent_name=example_single_tool_call_simple_agent \ +input_jsonl_fpath=weather_query.jsonl \ +output_jsonl_fpath=weather_rollouts.jsonl \ @@ -257,7 +256,7 @@ def _preprocess_rows_from_config(self, config: RolloutCollectionConfig) -> List[ validate_prompt_compatibility([row for _, _, row in raw_rows], prompt_cfg) raw_rows = [(idx, s, apply_prompt_to_row(row, prompt_cfg)) for idx, s, row in raw_rows] - # For ng_reward_profile to match rollouts to tasks + # For gym eval profile to match rollouts to tasks row_to_task_idx: Dict[str, int] = dict() task_idx_to_rollout_idx: Dict[int, int] = Counter() row_idxs_missing_agent_ref: List[int] = [] @@ -464,7 +463,7 @@ async def run_from_config(self, config: RolloutCollectionConfig) -> Tuple[List[D if config.disable_aggregation: print( "Skipping aggregate-metrics computation because disable_aggregation=True. " - "Run `ng_aggregate_rollouts` after all shards finish to compute the global metrics." + "Run `gym eval aggregate` after all shards finish to compute the global metrics." ) aggregate_metrics_fpath = None else: @@ -617,16 +616,9 @@ def setup_server_client( return server_client -def collect_rollouts(): # pragma: no cover - config = RolloutCollectionConfig.model_validate(get_global_config_dict()) - rch = RolloutCollectionHelper() - - asyncio.run(rch.run_from_config(config)) - - class RolloutAggregationConfig(BaseNeMoGymCLIConfig): """ - Aggregate metrics across rollout shards produced by `ng_collect_rollouts +disable_aggregation=true`. + Aggregate metrics across rollout shards produced by `gym eval run --no-serve +disable_aggregation=true`. Reads every JSONL file matching `input_glob`, computes aggregate metrics by POSTing to each agent server's `/aggregate_metrics` endpoint over the global union of records, and writes a @@ -636,7 +628,7 @@ class RolloutAggregationConfig(BaseNeMoGymCLIConfig): Examples: ```bash - ng_aggregate_rollouts \ + gym eval aggregate \ "+config_paths=[benchmarks/aime24/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ +input_glob='results/rollouts-rs*-chunk*.jsonl' \ +output_jsonl_fpath=results/rollouts.jsonl @@ -721,8 +713,15 @@ async def run_from_config(self, config: RolloutAggregationConfig) -> Optional[Pa return aggregate_metrics_fpath -def aggregate_rollouts(): # pragma: no cover - config = RolloutAggregationConfig.model_validate(get_global_config_dict()) - rah = RolloutAggregationHelper() +# Backward-compatibility shims (CLI refactor): these CLI entry points moved to `nemo_gym.cli.eval`. +# Re-exported lazily to avoid a circular import; accessing them emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 - asyncio.run(rah.run_from_config(config)) + +__getattr__ = moved_attr_getter( + __name__, + { + "collect_rollouts": "nemo_gym.cli.eval", + "aggregate_rollouts": "nemo_gym.cli.eval", + }, +) diff --git a/nemo_gym/server_status.py b/nemo_gym/server_status.py index cfa65eb067..2ab6b05a85 100644 --- a/nemo_gym/server_status.py +++ b/nemo_gym/server_status.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging from time import time from typing import List @@ -21,6 +22,9 @@ from nemo_gym.server_utils import ServerClient, ServerInstanceDisplayConfig, ServerStatus +logger = logging.getLogger(__name__) + + class StatusCommand: """Main class to check server status""" @@ -73,10 +77,10 @@ def discover_servers(self) -> List[ServerInstanceDisplayConfig]: return servers except (requests.RequestException, ConnectionError) as e: - print(f""" -Could not connect to head server: {e} -Is the head server running? Start it with: `ng_run` - """) + logger.warning( + "Could not connect to head server: %s. Is the head server running? Start it with: `gym env start`", + e, + ) return [] def display_status(self, servers: List[ServerInstanceDisplayConfig]) -> None: diff --git a/nemo_gym/train_data_utils.py b/nemo_gym/train_data_utils.py index 5191741b37..c632757e53 100644 --- a/nemo_gym/train_data_utils.py +++ b/nemo_gym/train_data_utils.py @@ -40,7 +40,6 @@ from nemo_gym.global_config import ( HF_TOKEN_KEY_NAME, GlobalConfigDictParser, - GlobalConfigDictParserConfig, get_global_config_dict, ) from nemo_gym.hf_utils import ( @@ -58,7 +57,7 @@ class TrainDataProcessorConfig(BaseNeMoGymCLIConfig): ```bash config_paths="resources_servers/example_multi_step/configs/example_multi_step.yaml,\\ responses_api_models/openai_model/configs/openai_model.yaml" - ng_prepare_data "+config_paths=[${config_paths}]" \ + gym dataset collate "+config_paths=[${config_paths}]" \ +output_dirpath=data/example_multi_step \ +mode=example_validation ``` @@ -825,12 +824,14 @@ def validate_backend_credentials(backend: str) -> tuple[bool, str]: return True, "" -def prepare_data(): # pragma: no cover - global_config_dict = get_global_config_dict( - global_config_dict_parser_config=GlobalConfigDictParserConfig( - initial_global_config_dict=GlobalConfigDictParserConfig.NO_MODEL_GLOBAL_CONFIG_DICT, - ) - ) +# Backward-compatibility shim (CLI refactor): this CLI entry point moved to `nemo_gym.cli.dataset`. +# Re-exported lazily to avoid a circular import; accessing it emits a DeprecationWarning. +from nemo_gym.cli._compat import moved_attr_getter # noqa: E402 - data_processor = TrainDataProcessor() - data_processor.run(global_config_dict) + +__getattr__ = moved_attr_getter( + __name__, + { + "prepare_data": "nemo_gym.cli.dataset", + }, +) diff --git a/pyproject.toml b/pyproject.toml index 74065cca9e..7f58e7eedb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -338,87 +338,93 @@ Download = "https://github.com/NVIDIA-NeMo/Gym/releases" Homepage = "https://github.com/NVIDIA-NeMo/Gym" [project.scripts] + +gym = "nemo_gym.cli.main:main" +ng = "nemo_gym.cli.main:main" + +########################## DEPRECATED COMMANDS ########################## + # Run/test scripts for servers. -nemo_gym_run = "nemo_gym.cli:run" -ng_run = "nemo_gym.cli:run" -nemo_gym_test = "nemo_gym.cli:test" -ng_test = "nemo_gym.cli:test" -nemo_gym_test_all = "nemo_gym.cli:test_all" -ng_test_all = "nemo_gym.cli:test_all" +nemo_gym_run = "nemo_gym.cli.legacy:main" +ng_run = "nemo_gym.cli.legacy:main" +nemo_gym_test = "nemo_gym.cli.legacy:main" +ng_test = "nemo_gym.cli.legacy:main" +nemo_gym_test_all = "nemo_gym.cli.legacy:main" +ng_test_all = "nemo_gym.cli.legacy:main" # Development convenience aliases. -nemo_gym_dev_test = "nemo_gym.cli:dev_test" -ng_dev_test = "nemo_gym.cli:dev_test" -nemo_gym_init_resources_server = "nemo_gym.cli:init_resources_server" -ng_init_resources_server = "nemo_gym.cli:init_resources_server" +nemo_gym_dev_test = "nemo_gym.cli.legacy:main" +ng_dev_test = "nemo_gym.cli.legacy:main" +nemo_gym_init_resources_server = "nemo_gym.cli.legacy:main" +ng_init_resources_server = "nemo_gym.cli.legacy:main" # Benchmarks -nemo_gym_list_benchmarks = "nemo_gym.benchmarks:list_benchmarks" -ng_list_benchmarks = "nemo_gym.benchmarks:list_benchmarks" -nemo_gym_prepare_benchmark = "nemo_gym.benchmarks:prepare_benchmark" -ng_prepare_benchmark = "nemo_gym.benchmarks:prepare_benchmark" +nemo_gym_list_benchmarks = "nemo_gym.cli.legacy:main" +ng_list_benchmarks = "nemo_gym.cli.legacy:main" +nemo_gym_prepare_benchmark = "nemo_gym.cli.legacy:main" +ng_prepare_benchmark = "nemo_gym.cli.legacy:main" # Rollout collection -nemo_gym_collect_rollouts = "nemo_gym.rollout_collection:collect_rollouts" -ng_collect_rollouts = "nemo_gym.rollout_collection:collect_rollouts" -nemo_gym_e2e_collect_rollouts = "nemo_gym.cli:e2e_rollout_collection" -ng_e2e_collect_rollouts = "nemo_gym.cli:e2e_rollout_collection" -nemo_gym_aggregate_rollouts = "nemo_gym.rollout_collection:aggregate_rollouts" -ng_aggregate_rollouts = "nemo_gym.rollout_collection:aggregate_rollouts" +nemo_gym_collect_rollouts = "nemo_gym.cli.legacy:main" +ng_collect_rollouts = "nemo_gym.cli.legacy:main" +nemo_gym_e2e_collect_rollouts = "nemo_gym.cli.legacy:main" +ng_e2e_collect_rollouts = "nemo_gym.cli.legacy:main" +nemo_gym_aggregate_rollouts = "nemo_gym.cli.legacy:main" +ng_aggregate_rollouts = "nemo_gym.cli.legacy:main" # Prompt materialization -nemo_gym_materialize_prompts = "nemo_gym.prompt:materialize_prompts_cli" -ng_materialize_prompts = "nemo_gym.prompt:materialize_prompts_cli" +nemo_gym_materialize_prompts = "nemo_gym.cli.legacy:main" +ng_materialize_prompts = "nemo_gym.cli.legacy:main" # Reward profiling -nemo_gym_reward_profile = "nemo_gym.reward_profile:reward_profile" -ng_reward_profile = "nemo_gym.reward_profile:reward_profile" +nemo_gym_reward_profile = "nemo_gym.cli.legacy:main" +ng_reward_profile = "nemo_gym.cli.legacy:main" # Dataset management -nemo_gym_upload_dataset_to_gitlab = "nemo_gym.gitlab_utils:upload_jsonl_dataset_cli" -ng_upload_dataset_to_gitlab = "nemo_gym.gitlab_utils:upload_jsonl_dataset_cli" -nemo_gym_download_dataset_from_gitlab = "nemo_gym.gitlab_utils:download_jsonl_dataset_cli" -ng_download_dataset_from_gitlab = "nemo_gym.gitlab_utils:download_jsonl_dataset_cli" -nemo_gym_prepare_data = "nemo_gym.train_data_utils:prepare_data" -ng_prepare_data = "nemo_gym.train_data_utils:prepare_data" +nemo_gym_upload_dataset_to_gitlab = "nemo_gym.cli.legacy:main" +ng_upload_dataset_to_gitlab = "nemo_gym.cli.legacy:main" +nemo_gym_download_dataset_from_gitlab = "nemo_gym.cli.legacy:main" +ng_download_dataset_from_gitlab = "nemo_gym.cli.legacy:main" +nemo_gym_prepare_data = "nemo_gym.cli.legacy:main" +ng_prepare_data = "nemo_gym.cli.legacy:main" # HF dataset upload and download -nemo_gym_upload_dataset_to_hf = "nemo_gym.dataset_orchestrator:upload_jsonl_dataset_to_hf_cli" -ng_upload_dataset_to_hf = "nemo_gym.dataset_orchestrator:upload_jsonl_dataset_to_hf_cli" -nemo_gym_download_dataset_from_hf = "nemo_gym.dataset_orchestrator:download_jsonl_dataset_from_hf_cli" -ng_download_dataset_from_hf = "nemo_gym.dataset_orchestrator:download_jsonl_dataset_from_hf_cli" +nemo_gym_upload_dataset_to_hf = "nemo_gym.cli.legacy:main" +ng_upload_dataset_to_hf = "nemo_gym.cli.legacy:main" +nemo_gym_download_dataset_from_hf = "nemo_gym.cli.legacy:main" +ng_download_dataset_from_hf = "nemo_gym.cli.legacy:main" # Gitlab -> HF (upload then delete) -nemo_gym_gitlab_to_hf_dataset = "nemo_gym.dataset_orchestrator:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli" -ng_gitlab_to_hf_dataset = "nemo_gym.dataset_orchestrator:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli" +nemo_gym_gitlab_to_hf_dataset = "nemo_gym.cli.legacy:main" +ng_gitlab_to_hf_dataset = "nemo_gym.cli.legacy:main" # Manually delete from Gitlab -nemo_gym_delete_dataset_from_gitlab = "nemo_gym.dataset_orchestrator:delete_jsonl_dataset_from_gitlab_cli" -ng_delete_dataset_from_gitlab = "nemo_gym.dataset_orchestrator:delete_jsonl_dataset_from_gitlab_cli" +nemo_gym_delete_dataset_from_gitlab = "nemo_gym.cli.legacy:main" +ng_delete_dataset_from_gitlab = "nemo_gym.cli.legacy:main" # Configuration utils -nemo_gym_dump_config = "nemo_gym.cli:dump_config" -ng_dump_config = "nemo_gym.cli:dump_config" +nemo_gym_dump_config = "nemo_gym.cli.legacy:main" +ng_dump_config = "nemo_gym.cli.legacy:main" # Display help -nemo_gym_help = "nemo_gym.cli:display_help" -ng_help = "nemo_gym.cli:display_help" +nemo_gym_help = "nemo_gym.cli.legacy:main" +ng_help = "nemo_gym.cli.legacy:main" # Server status -nemo_gym_status = "nemo_gym.cli:status" -ng_status = "nemo_gym.cli:status" +nemo_gym_status = "nemo_gym.cli.legacy:main" +ng_status = "nemo_gym.cli.legacy:main" # Environment-specific uv pip list -nemo_gym_pip_list = "nemo_gym.cli:pip_list" -ng_pip_list = "nemo_gym.cli:pip_list" +nemo_gym_pip_list = "nemo_gym.cli.legacy:main" +ng_pip_list = "nemo_gym.cli.legacy:main" # Display version -nemo_gym_version = "nemo_gym.cli:version" -ng_version = "nemo_gym.cli:version" +nemo_gym_version = "nemo_gym.cli.legacy:main" +ng_version = "nemo_gym.cli.legacy:main" # Re-install Gym and dependencies -nemo_gym_reinstall = "nemo_gym.cli:reinstall" -ng_reinstall = "nemo_gym.cli:reinstall" +nemo_gym_reinstall = "nemo_gym.cli.legacy:main" +ng_reinstall = "nemo_gym.cli.legacy:main" [tool.setuptools.packages.find] where = ["."] diff --git a/resources_servers/abstention/README.md b/resources_servers/abstention/README.md index c790f869fb..a7e11c9851 100644 --- a/resources_servers/abstention/README.md +++ b/resources_servers/abstention/README.md @@ -15,20 +15,20 @@ The dataset used is [HotPotQA](https://hotpotqa.github.io/) (fullwiki split). ## Running servers ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -resources_servers/abstention/configs/abstention.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --model-type openai_model \ + --resources-server abstention \ +abstention.resources_servers.abstention.judge_model_server.name=policy_model ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=abstention_simple_agent \ - +input_jsonl_fpath=resources_servers/abstention/data/example.jsonl \ - +output_jsonl_fpath=results/abstention_verify_responses.jsonl \ - +limit=3 +gym eval run --no-serve \ + --agent abstention_simple_agent \ + --input resources_servers/abstention/data/example.jsonl \ + --output results/abstention_verify_responses.jsonl \ + --limit 3 ``` ## Preprocessing HotPotQA data diff --git a/resources_servers/arc_agi/README.md b/resources_servers/arc_agi/README.md index 08987a922b..463f058517 100644 --- a/resources_servers/arc_agi/README.md +++ b/resources_servers/arc_agi/README.md @@ -51,7 +51,9 @@ uv sync ### Start ARC-AGI environment (we can reuse the same one for ARC-AGI-1 and 2): ```bash -ng_run "+config_paths=[resources_servers/arc_agi/configs/arc_agi.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --resources-server arc_agi \ + --model-type vllm_model ``` @@ -59,12 +61,18 @@ ng_run "+config_paths=[resources_servers/arc_agi/configs/arc_agi.yaml,responses_ ARC-AGI-1 example rollouts: ```bash -ng_collect_rollouts +agent_name=arc_agi_simple_agent +input_jsonl_fpath=resources_servers/arc_agi/data/example_1.jsonl +output_jsonl_fpath=resources_servers/arc_agi/data/example_1_rollouts.jsonl +gym eval run --no-serve \ + --agent arc_agi_simple_agent \ + --input resources_servers/arc_agi/data/example_1.jsonl \ + --output resources_servers/arc_agi/data/example_1_rollouts.jsonl ``` ARC-AGI-2 example rollouts: ```bash -ng_collect_rollouts +agent_name=arc_agi_simple_agent +input_jsonl_fpath=resources_servers/arc_agi/data/example_2.jsonl +output_jsonl_fpath=resources_servers/arc_agi/data/example_2_rollouts.jsonl +gym eval run --no-serve \ + --agent arc_agi_simple_agent \ + --input resources_servers/arc_agi/data/example_2.jsonl \ + --output resources_servers/arc_agi/data/example_2_rollouts.jsonl ``` For training, see the [docs](https://docs.nvidia.com/nemo/gym/latest/training-tutorials/nemo-rl-grpo/index.html). \ No newline at end of file diff --git a/resources_servers/arena_judge/README.md b/resources_servers/arena_judge/README.md index 83caf6521f..65fcb175b7 100644 --- a/resources_servers/arena_judge/README.md +++ b/resources_servers/arena_judge/README.md @@ -47,16 +47,16 @@ Each JSONL row must carry the following top-level fields (pydantic ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/arena_judge/configs/arena_judge.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server arena_judge # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=arena_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/arena_judge/data/example.jsonl \ - +output_jsonl_fpath=results/arena_judge_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent arena_judge_simple_agent \ + --input resources_servers/arena_judge/data/example.jsonl \ + --output results/arena_judge_rollouts.jsonl \ + --num-repeats 1 ``` ## Configuring the judge diff --git a/resources_servers/asr_with_pc/README.md b/resources_servers/asr_with_pc/README.md index 87430f1115..cb9d0e56b0 100644 --- a/resources_servers/asr_with_pc/README.md +++ b/resources_servers/asr_with_pc/README.md @@ -50,19 +50,19 @@ workaround until the schema is extended. ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/asr_with_pc/configs/asr_with_pc.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server asr_with_pc ``` ## Collecting rollouts (5-example smoke test) ```bash -ng_collect_rollouts \ - +agent_name=asr_with_pc_simple_agent \ - +input_jsonl_fpath=resources_servers/asr_with_pc/data/example.jsonl \ - +output_jsonl_fpath=results/asr_with_pc_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent asr_with_pc_simple_agent \ + --input resources_servers/asr_with_pc/data/example.jsonl \ + --output results/asr_with_pc_rollouts.jsonl \ + --num-repeats 1 ``` ## Regenerating example data diff --git a/resources_servers/aviary/README.md b/resources_servers/aviary/README.md index 83307c67de..efe706ff5b 100644 --- a/resources_servers/aviary/README.md +++ b/resources_servers/aviary/README.md @@ -22,17 +22,18 @@ This resources server adapts [Aviary environments](https://github.com/Future-Hou Run the GSM8K Aviary resources server together with a model config: ```bash -config_paths="resources_servers/aviary/configs/gsm8k_aviary.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --resources-server aviary/gsm8k_aviary \ + --model-type vllm_model ``` Then collect rollouts: ```bash -ng_collect_rollouts \ - +agent_name=gsm8k_aviary_agent +input_jsonl_fpath=resources_servers/aviary/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/aviary/data/example_rollouts.jsonl +gym eval run --no-serve \ + --agent gsm8k_aviary_agent \ + --input resources_servers/aviary/data/example.jsonl \ + --output resources_servers/aviary/data/example_rollouts.jsonl ``` # BixBench-Hypothesis (BBH) @@ -48,17 +49,18 @@ Then, prepare your Gym data with the task_idx values of the problems you would l Once the dataset server is running and is accessible at a specific URL, update your config based on [configs/bbh_remote.yaml](configs/bbh_remote.yaml) with the server URL and api key, and launch NeMo-Gym as follows: ```bash -config_paths="resources_servers/aviary/configs/bbh_remote.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --resources-server aviary/bbh_remote \ + --model-type vllm_model ``` -Then collect rollouts on your data as follows (updating the input_jsonl_fpath to your Gym data if needed): +Then collect rollouts on your data as follows (updating the input file to your Gym data if needed): ```bash -ng_collect_rollouts \ - +agent_name=bbh_aviary_agent +input_jsonl_fpath=resources_servers/aviary/data/bbh_train_example.jsonl \ - +output_jsonl_fpath=resources_servers/aviary/data/example_bbh_rollouts.jsonl +gym eval run --no-serve \ + --agent bbh_aviary_agent \ + --input resources_servers/aviary/data/bbh_train_example.jsonl \ + --output resources_servers/aviary/data/example_bbh_rollouts.jsonl ``` To run training with NeMo-RL, set the following fields in your NeMo-RL container (where train_data.jsonl and validation_data.jsonl are set to your train/val Gym data respectively, and bbh_remote.yaml is updated with your dataset server URL/api-key): @@ -98,14 +100,15 @@ cd /path/to/gym/directory ``` And then bring up NeMo-Gym: ```bash -config_paths="resources_servers/aviary/configs/bbh_bundled.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --resources-server aviary/bbh_bundled \ + --model-type vllm_model ``` ```bash -ng_collect_rollouts \ - +agent_name=bbh_aviary_agent +input_jsonl_fpath=resources_servers/aviary/data/bbh_train_example.jsonl \ - +output_jsonl_fpath=resources_servers/aviary/data/example_bbh_rollouts.jsonl +gym eval run --no-serve \ + --agent bbh_aviary_agent \ + --input resources_servers/aviary/data/bbh_train_example.jsonl \ + --output resources_servers/aviary/data/example_bbh_rollouts.jsonl ``` If you are running training with NeMo-RL and NeMo-Gym, add the following modification to your `ray.sub` file in NeMo-RL to support adding a setup command: diff --git a/resources_servers/bigcodebench/README.md b/resources_servers/bigcodebench/README.md index 7a177cd7ef..5948f09433 100644 --- a/resources_servers/bigcodebench/README.md +++ b/resources_servers/bigcodebench/README.md @@ -23,16 +23,16 @@ starts are instant. ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/bigcodebench/configs/bigcodebench.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server bigcodebench # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=bigcodebench_simple_agent \ - +input_jsonl_fpath=resources_servers/bigcodebench/data/example.jsonl \ - +output_jsonl_fpath=results/bigcodebench_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent bigcodebench_simple_agent \ + --input resources_servers/bigcodebench/data/example.jsonl \ + --output results/bigcodebench_rollouts.jsonl \ + --num-repeats 1 ``` ## Reasoning-parser note diff --git a/resources_servers/bird_sql/README.md b/resources_servers/bird_sql/README.md index 8921249e3a..78b7b10a6d 100644 --- a/resources_servers/bird_sql/README.md +++ b/resources_servers/bird_sql/README.md @@ -34,9 +34,9 @@ ensure_bird_sql() ### Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/bird_sql/configs/bird_sql.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server bird_sql ``` Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in @@ -45,11 +45,11 @@ Requires `policy_base_url` / `policy_api_key` / `policy_model_name` in ### Collecting rollouts (5-example smoke test) ```bash -ng_collect_rollouts \ - +agent_name=bird_sql_simple_agent \ - +input_jsonl_fpath=resources_servers/bird_sql/data/example.jsonl \ - +output_jsonl_fpath=results/bird_sql_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent bird_sql_simple_agent \ + --input resources_servers/bird_sql/data/example.jsonl \ + --output results/bird_sql_rollouts.jsonl \ + --num-repeats 1 ``` For a full BIRD dev run, see `benchmarks/birdbench/README.md`. diff --git a/resources_servers/bird_sql/eval_utils.py b/resources_servers/bird_sql/eval_utils.py index 7125379800..3acac2eb7f 100644 --- a/resources_servers/bird_sql/eval_utils.py +++ b/resources_servers/bird_sql/eval_utils.py @@ -8,7 +8,7 @@ SQL execution uses ``asyncio.to_thread`` rather than Ray remote tasks. When this resource server runs alongside a Ray-coordinated multi-node DP -vLLM (Gym's ``ng_run`` attaches to the same Ray cluster), the vLLM engines +vLLM (``gym env start`` attaches to the same Ray cluster), the vLLM engines consume the Ray slots and ``@ray.remote`` sqlite tasks sit in the scheduler queue past the per-query timeout, get cancelled, and every rollout reports ``gold_execution_error``. SQLite queries are fast and self-contained — no diff --git a/resources_servers/blackjack/README.md b/resources_servers/blackjack/README.md index 54fbff84ad..aa724dcd21 100644 --- a/resources_servers/blackjack/README.md +++ b/resources_servers/blackjack/README.md @@ -9,7 +9,9 @@ Example data provided in `data/example.jsonl` (system prompt only, no verifier_m ## Run ```bash -ng_run "+config_paths=[resources_servers/blackjack/configs/blackjack.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --resources-server blackjack \ + --model-type vllm_model ``` ## Data @@ -19,8 +21,8 @@ Each game is generated on the fly during `reset()`, so every row in `example.jso ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=blackjack_gymnasium_agent \ - +input_jsonl_fpath=resources_servers/blackjack/data/example.jsonl \ - +output_jsonl_fpath=results/blackjack_rollouts.jsonl +gym eval run --no-serve \ + --agent blackjack_gymnasium_agent \ + --input resources_servers/blackjack/data/example.jsonl \ + --output results/blackjack_rollouts.jsonl ``` diff --git a/resources_servers/browsecomp_advanced_harness/README.md b/resources_servers/browsecomp_advanced_harness/README.md index 8b2917f32e..2e718b0a56 100644 --- a/resources_servers/browsecomp_advanced_harness/README.md +++ b/resources_servers/browsecomp_advanced_harness/README.md @@ -29,10 +29,9 @@ judge_model_name: Qwen/Qwen3-235B-A22B-Instruct-2507 ```bash # If you want to run with browsecomp benchmark instead of the example samples, need to change the datasets part to the one like `benchmarks/browsecomp/config.yaml` in `resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml` -config_paths="resources_servers/browsecomp_advanced_harness/configs/browsecomp_advanced_harness.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" - -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --resources-server browsecomp_advanced_harness \ + --model-type vllm_model ``` diff --git a/resources_servers/calendar/README.md b/resources_servers/calendar/README.md index c65fa6a916..07ef641f5d 100644 --- a/resources_servers/calendar/README.md +++ b/resources_servers/calendar/README.md @@ -16,9 +16,9 @@ The conversations in the dataset are generated using personas from the [nvidia/N The following is an example command for running this resources server along with an OpenAI model: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -resources_servers/calendar/configs/calendar.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --resources-server calendar ``` ## Collecting rollouts @@ -26,11 +26,11 @@ ng_run "+config_paths=[$config_paths]" Rollouts can be collected using the example dataset as follows: ```bash -ng_collect_rollouts \ - +agent_name=calendar_agent \ - +input_jsonl_fpath=data/example.jsonl \ - +output_jsonl_fpath=results/example_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent calendar_agent \ + --input data/example.jsonl \ + --output results/example_rollouts.jsonl \ + --limit 5 ``` The input JSONL file should contain entries with: diff --git a/resources_servers/circle_click/README.md b/resources_servers/circle_click/README.md index 3fcfe8819f..610dc63eb8 100644 --- a/resources_servers/circle_click/README.md +++ b/resources_servers/circle_click/README.md @@ -12,8 +12,14 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & -ng_run "+config_paths=[resources_servers/circle_click/configs/circle_click.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" & -ng_collect_rollouts +agent_name=circle_click_simple_agent +input_jsonl_fpath=resources_servers/circle_click/data/example.jsonl +output_jsonl_fpath=resources_servers/circle_click/data/example_rollouts.jsonl +limit=1 +gym env start \ + --resources-server circle_click \ + --model-type vllm_model & +gym eval run --no-serve \ + --agent circle_click_simple_agent \ + --input resources_servers/circle_click/data/example.jsonl \ + --output resources_servers/circle_click/data/example_rollouts.jsonl \ + --limit 1 ``` # Generating Data diff --git a/resources_servers/circle_count/README.md b/resources_servers/circle_count/README.md index 5d567b04dc..e30908513e 100644 --- a/resources_servers/circle_count/README.md +++ b/resources_servers/circle_count/README.md @@ -19,8 +19,14 @@ policy_model_name: Qwen/Qwen3-VL-8B-Instruct ```bash vllm serve Qwen/Qwen3-VL-8B-Instruct -tp 8 --enable-auto-tool-choice --tool-call-parser hermes & -ng_run "+config_paths=[resources_servers/circle_count/configs/circle_count.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" & -ng_collect_rollouts +agent_name=circle_count_simple_agent +input_jsonl_fpath=resources_servers/circle_count/data/example.jsonl +output_jsonl_fpath=resources_servers/circle_count/data/example_rollouts.jsonl +limit=1 +gym env start \ + --resources-server circle_count \ + --model-type vllm_model & +gym eval run --no-serve \ + --agent circle_count_simple_agent \ + --input resources_servers/circle_count/data/example.jsonl \ + --output resources_servers/circle_count/data/example_rollouts.jsonl \ + --limit 1 ``` # Generating Data diff --git a/resources_servers/code_fim/README.md b/resources_servers/code_fim/README.md index 809548160e..5e89306bc7 100644 --- a/resources_servers/code_fim/README.md +++ b/resources_servers/code_fim/README.md @@ -58,16 +58,16 @@ the reward; pass@k / majority@k are computed by the metrics layer. ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/code_fim/configs/code_fim.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server code_fim # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=code_fim_simple_agent \ - +input_jsonl_fpath=resources_servers/code_fim/data/example.jsonl \ - +output_jsonl_fpath=results/code_fim_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent code_fim_simple_agent \ + --input resources_servers/code_fim/data/example.jsonl \ + --output results/code_fim_rollouts.jsonl \ + --num-repeats 1 ``` Start vLLM with `--reasoning-parser ` (e.g. `deepseek_r1` for diff --git a/resources_servers/code_gen/README.md b/resources_servers/code_gen/README.md index bdc2039890..fc0f0dff43 100644 --- a/resources_servers/code_gen/README.md +++ b/resources_servers/code_gen/README.md @@ -24,15 +24,16 @@ We use the LiveCodeBench execution code. ```bash # Running the server -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/code_gen/configs/code_gen.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --resources-server code_gen # Collect rollouts from example problems -ng_collect_rollouts +agent_name=code_gen_simple_agent \ - +input_jsonl_fpath=resources_servers/code_gen/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/code_gen/data/example_rollouts.jsonl \ - +limit=null +gym eval run --no-serve \ + --agent code_gen_simple_agent \ + --input resources_servers/code_gen/data/example.jsonl \ + --output resources_servers/code_gen/data/example_rollouts.jsonl \ + --limit null ``` ## Licensing information diff --git a/resources_servers/code_gen/livecodebench_accuracy_test.py b/resources_servers/code_gen/livecodebench_accuracy_test.py index 25562a0acb..d8e4011f66 100644 --- a/resources_servers/code_gen/livecodebench_accuracy_test.py +++ b/resources_servers/code_gen/livecodebench_accuracy_test.py @@ -26,7 +26,7 @@ ```bash config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ resources_servers/code_gen/configs/code_gen.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start "+config_paths=[${config_paths}]" ``` """ diff --git a/resources_servers/code_gen/livecodebench_accuracy_test_prep.py b/resources_servers/code_gen/livecodebench_accuracy_test_prep.py index 49db32c4a9..beb2c68d4a 100644 --- a/resources_servers/code_gen/livecodebench_accuracy_test_prep.py +++ b/resources_servers/code_gen/livecodebench_accuracy_test_prep.py @@ -57,13 +57,13 @@ Download the test data using: ```bash -ng_download_dataset_from_gitlab \ +gym dataset download --storage gitlab \ +dataset_name=livecodebench \ +version=0.0.1 \ +artifact_fpath=gpt-4o-2024-05-13_Scenario.codegeneration_10_0.2_eval_all.json \ +output_fpath=resources_servers/code_gen/data/gpt-4o-2024-05-13_Scenario.codegeneration_10_0.2_eval_all.json -ng_download_dataset_from_gitlab \ +gym dataset download --storage gitlab \ +dataset_name=livecodebench \ +version=0.0.1 \ +artifact_fpath=livecodebench_prompts.jsonl \ diff --git a/resources_servers/code_gen/scripts/preprocess_train_dataset.py b/resources_servers/code_gen/scripts/preprocess_train_dataset.py index 429079910a..3c50bd16f6 100644 --- a/resources_servers/code_gen/scripts/preprocess_train_dataset.py +++ b/resources_servers/code_gen/scripts/preprocess_train_dataset.py @@ -23,7 +23,7 @@ Upload: ```bash -ng_upload_dataset_to_gitlab \ +gym dataset upload --storage gitlab \ +dataset_name=opencodereasoning_filtered \ +version=0.0.1 \ +input_jsonl_fpath=resources_servers/code_gen/data/opencodereasoning_filtered_25k_train.jsonl @@ -31,7 +31,7 @@ Rollout collection. We match the LCB setting for reward profiling. For gpt-4o-2024-05-13 this should be around 33%. ```bash -ng_collect_rollouts +agent_name=code_gen_simple_agent \ +gym eval run --no-serve +agent_name=code_gen_simple_agent \ +input_jsonl_fpath=resources_servers/code_gen/data/opencodereasoning_filtered_25k_train.jsonl \ +output_jsonl_fpath=resources_servers/code_gen/data/opencodereasoning_filtered_25k_train_1k_gpt-4o-2024-05-13_rollouts.jsonl \ +responses_create_params.temperature=0.2 \ diff --git a/resources_servers/competitive_coding_challenges/README.md b/resources_servers/competitive_coding_challenges/README.md index ade3fcf957..962d8e125b 100644 --- a/resources_servers/competitive_coding_challenges/README.md +++ b/resources_servers/competitive_coding_challenges/README.md @@ -61,15 +61,17 @@ elsewhere. Cluster/SLURM users can co-launch the sandbox via Skills' The following allows users to launch the environment followed by the request for generating and collecting rollouts. ```bash -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --config "$config_paths" \ +simple_agent.responses_api_agents.simple_agent.resources_server.name=competitive_coding_challenges_resources_server -ng_collect_rollouts +agent_name=simple_agent \ - +input_jsonl_fpath=$input_jsonl_fpath \ - +output_jsonl_fpath=$output_jsonl_fpath \ - +limit=null \ - +num_repeats=1 \ - +num_samples_in_parallel=null +gym eval run --no-serve \ + --agent simple_agent \ + --input $input_jsonl_fpath \ + --output $output_jsonl_fpath \ + --limit null \ + --num-repeats 1 \ + --concurrency null ``` ## Additional Information diff --git a/resources_servers/cvdp/README.md b/resources_servers/cvdp/README.md index 67db483f47..8df29b144d 100644 --- a/resources_servers/cvdp/README.md +++ b/resources_servers/cvdp/README.md @@ -123,7 +123,9 @@ apt install -y ./apptainer_1.3.1_amd64.deb ### Step 1 — Start servers ```bash -ng_run "+config_paths=[resources_servers/cvdp/configs/cvdp.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --resources-server cvdp \ + --model-type vllm_model ``` ### Step 2 — Run rollout collection @@ -131,15 +133,18 @@ ng_run "+config_paths=[resources_servers/cvdp/configs/cvdp.yaml,responses_api_mo `num_repeats` controls how many times each problem is run (for pass@k metrics): ```bash -ng_collect_rollouts \ - +agent_name=cvdp_agent \ - +input_jsonl_fpath=resources_servers/cvdp/data/.jsonl \ - +output_jsonl_fpath=results/rollouts.jsonl \ - +num_repeats=5 \ - +num_samples_in_parallel=4 \ - "+responses_create_params={max_output_tokens: 4096, temperature: 0.2, top_p: 0.7}" \ - "+config_paths=[resources_servers/cvdp/configs/cvdp.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" - "+resume_from_cache=True" +gym eval run --no-serve \ + --agent cvdp_agent \ + --input resources_servers/cvdp/data/.jsonl \ + --output results/rollouts.jsonl \ + --num-repeats 5 \ + --concurrency 4 \ + --max-output-tokens 4096 \ + --temperature 0.2 \ + --top-p 0.7 \ + --resources-server cvdp \ + --model-type vllm_model \ + --resume ``` At the end of this step, you should have rollouts.jsonl, rollouts_agent_metrics.json, rollouts_materialized_inputs.jsonl, rollouts_reward_profiling.jsonl. @@ -158,7 +163,7 @@ python resources_servers/cvdp/scripts/cvdp_pass_at_k_report.py \ | Arg | Required | Default | Description | | ------------ | -------- | ---------- | --------------------------------------------------------- | -| `--rollouts` | Yes | — | Rollout JSONL from `ng_collect_rollouts` | +| `--rollouts` | Yes | — | Rollout JSONL from `gym eval run --no-serve` | | `--output` | Yes | — | Output directory for reports | | `--model` | No | `nemo-gym` | Model name shown in report metadata | | `--dataset` | No | — | Path to original CVDP dataset JSONL (for report metadata) | diff --git a/resources_servers/cvdp/scripts/cvdp_pass_at_k_report.py b/resources_servers/cvdp/scripts/cvdp_pass_at_k_report.py index 6d50b33c8a..2c25cb9b4c 100644 --- a/resources_servers/cvdp/scripts/cvdp_pass_at_k_report.py +++ b/resources_servers/cvdp/scripts/cvdp_pass_at_k_report.py @@ -47,7 +47,7 @@ def split_by_rollout(rollouts_path: str) -> dict[int, list[dict]]: def main() -> None: parser = argparse.ArgumentParser(description="Generate CVDP pass@k report from NeMo-Gym rollouts") - parser.add_argument("--rollouts", required=True, help="pass@k rollout JSONL from ng_collect_rollouts") + parser.add_argument("--rollouts", required=True, help="pass@k rollout JSONL from gym eval run") parser.add_argument("--output", required=True, help="Output directory") parser.add_argument("--model", default="nemo-gym", help="Model name for report metadata") parser.add_argument("--dataset", help="Path to original CVDP dataset JSONL (for report metadata)") diff --git a/resources_servers/equivalence_llm_judge/README.md b/resources_servers/equivalence_llm_judge/README.md index f30a1bc582..a7881a27a7 100644 --- a/resources_servers/equivalence_llm_judge/README.md +++ b/resources_servers/equivalence_llm_judge/README.md @@ -51,10 +51,9 @@ equivalence_llm_judge_simple_agent: ### Usage Spin up with a judge model and prompt: ```bash -config_paths="resources_servers/equivalence_llm_judge/configs/equivalence_llm_judge.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --resources-server equivalence_llm_judge \ + --model-type openai_model \ +equivalence_llm_judge.resources_servers.equivalence_llm_judge.judge_responses_create_params.max_output_tokens=256 \ +equivalence_llm_judge.resources_servers.equivalence_llm_judge.judge_system_message="You are a careful arbiter." \ "+equivalence_llm_judge.resources_servers.equivalence_llm_judge.judge_prompt_template='<|Problem|>\n{question}\n\n<|Gold|>\n{expected_answer}\n\n<|Prediction|>\n{generated_answer}\n'" diff --git a/resources_servers/ether0/README.md b/resources_servers/ether0/README.md index f84ab52ccf..38a43eff8b 100644 --- a/resources_servers/ether0/README.md +++ b/resources_servers/ether0/README.md @@ -27,13 +27,15 @@ Start servers and collect rollouts ```bash # start vllm and nemo gym servers vllm serve futurehouse/ether0 & -ng_run "+config_paths=[resources_servers/ether0/configs/ether0.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" & +gym env start \ + --resources-server ether0 \ + --model-type vllm_model & # wait for above to be ready -ng_collect_rollouts \ - +agent_name=ether0_simple_agent \ - +input_jsonl_fpath=resources_servers/ether0/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/ether0/data/ether0_rollouts.jsonl +gym eval run --no-serve \ + --agent ether0_simple_agent \ + --input resources_servers/ether0/data/example.jsonl \ + --output resources_servers/ether0/data/ether0_rollouts.jsonl tail -n 1 resources_servers/ether0/data/ether0_rollouts.jsonl | jq | less ``` diff --git a/resources_servers/evalplus/README.md b/resources_servers/evalplus/README.md index d7b6186b00..f07b850e58 100644 --- a/resources_servers/evalplus/README.md +++ b/resources_servers/evalplus/README.md @@ -46,16 +46,16 @@ compute pass@k for each separately. ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/evalplus/configs/evalplus.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server evalplus # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=evalplus_simple_agent \ - +input_jsonl_fpath=resources_servers/evalplus/data/example.jsonl \ - +output_jsonl_fpath=results/evalplus_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent evalplus_simple_agent \ + --input resources_servers/evalplus/data/example.jsonl \ + --output results/evalplus_rollouts.jsonl \ + --num-repeats 1 ``` Start vLLM with `--reasoning-parser ` (e.g. `deepseek_r1` for diff --git a/resources_servers/example_multi_turn_gymnasium/README.md b/resources_servers/example_multi_turn_gymnasium/README.md index 68d1930157..6d71a88f1a 100644 --- a/resources_servers/example_multi_turn_gymnasium/README.md +++ b/resources_servers/example_multi_turn_gymnasium/README.md @@ -15,14 +15,16 @@ Example data provided in `data/example.jsonl`. ## Run ```bash -ng_run "+config_paths=[resources_servers/example_multi_turn_gymnasium/configs/example_multi_turn_gymnasium.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --resources-server example_multi_turn_gymnasium \ + --model-type vllm_model ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=example_multi_turn_gymnasium_agent \ - +input_jsonl_fpath=resources_servers/example_multi_turn_gymnasium/data/example.jsonl \ - +output_jsonl_fpath=results/example_multi_turn_rollouts.jsonl +gym eval run --no-serve \ + --agent example_multi_turn_gymnasium_agent \ + --input resources_servers/example_multi_turn_gymnasium/data/example.jsonl \ + --output results/example_multi_turn_rollouts.jsonl ``` diff --git a/resources_servers/finance_sec_search/README.md b/resources_servers/finance_sec_search/README.md index 6c7afdb064..46958307b0 100644 --- a/resources_servers/finance_sec_search/README.md +++ b/resources_servers/finance_sec_search/README.md @@ -152,8 +152,8 @@ public benchmark lives in `benchmarks/finance_sec_search/`. It downloads the `public.csv` dataset from GitHub and converts it to Gym format: ```bash -# Prepare via Gym CLI (recommended — used by ng_run with benchmark configs): -ng_prepare_benchmark +benchmark=benchmarks/finance_sec_search/config_no_web_search.yaml +# Prepare via Gym CLI (recommended — used by gym env start with benchmark configs): +gym eval prepare --benchmark finance_sec_search/config_no_web_search # Or run the script directly: python benchmarks/finance_sec_search/prepare.py # without web_search @@ -194,40 +194,42 @@ Launch a vLLM-compatible model server (e.g. Qwen3-30B-A3B) so the policy and jud With a local vLLM model server: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,resources_servers/finance_sec_search/configs/finance_sec_search.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server finance_sec_search ``` Or with an OpenAI-compatible API (e.g. OpenAI, Azure, NIM): ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,resources_servers/finance_sec_search/configs/finance_sec_search.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --resources-server finance_sec_search ``` ### 4. Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=finance_agent \ - +input_jsonl_fpath=resources_servers/finance_sec_search/data/example.jsonl \ - +output_jsonl_fpath=results/finance_sec_search_rollouts.jsonl +gym eval run --no-serve \ + --agent finance_agent \ + --input resources_servers/finance_sec_search/data/example.jsonl \ + --output results/finance_sec_search_rollouts.jsonl ``` -Add `+limit=1` for a quick single-question test: +Add `--limit 1` for a quick single-question test: ```bash -ng_collect_rollouts \ - +agent_name=finance_agent \ - +input_jsonl_fpath=resources_servers/finance_sec_search/data/example.jsonl \ - +output_jsonl_fpath=results/finance_sec_search_rollouts.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent finance_agent \ + --input resources_servers/finance_sec_search/data/example.jsonl \ + --output results/finance_sec_search_rollouts.jsonl \ + --limit 1 ``` ### Run tests ```bash -ng_test +entrypoint=resources_servers/finance_sec_search +gym env test --resources-server finance_sec_search ``` ## Verification diff --git a/resources_servers/format_verification/README.md b/resources_servers/format_verification/README.md index 978983216f..ade87834f6 100644 --- a/resources_servers/format_verification/README.md +++ b/resources_servers/format_verification/README.md @@ -26,63 +26,63 @@ See [ARCHITECTURE.md](ARCHITECTURE.md) for system diagrams and detailed field ma ### Freeform Formatting ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/format_verification/configs/freeform_formatting.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --model-type openai_model \ + --resources-server format_verification/freeform_formatting ``` Collect rollouts: ```bash -ng_collect_rollouts \ - +agent_name=freeform_formatting_simple_agent \ - +input_jsonl_fpath=resources_servers/format_verification/data/ds2_freeform_formatting_train.jsonl \ - +output_jsonl_fpath=results/freeform_formatting_rollouts.jsonl \ - +resume_from_cache=True \ - +num_samples_in_parallel=256 +gym eval run --no-serve \ + --agent freeform_formatting_simple_agent \ + --input resources_servers/format_verification/data/ds2_freeform_formatting_train.jsonl \ + --output results/freeform_formatting_rollouts.jsonl \ + --concurrency 256 \ + --resume ``` ### Citation Format ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/format_verification/configs/citation_format.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --model-type openai_model \ + --resources-server format_verification/citation_format ``` Collect rollouts: ```bash -ng_collect_rollouts \ - +agent_name=citation_format_simple_agent \ - +input_jsonl_fpath=resources_servers/format_verification/data/ds3_citation_format_train.jsonl \ - +output_jsonl_fpath=results/citation_format_rollouts.jsonl \ - +resume_from_cache=True \ - +num_samples_in_parallel=256 +gym eval run --no-serve \ + --agent citation_format_simple_agent \ + --input resources_servers/format_verification/data/ds3_citation_format_train.jsonl \ + --output results/citation_format_rollouts.jsonl \ + --concurrency 256 \ + --resume ``` ## Downloading Data ### Freeform Formatting ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/format_verification/configs/freeform_formatting.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/format_verification_freeform/ \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resources-server format_verification/freeform_formatting \ + --output-dir data/format_verification_freeform/ \ + --mode train_preparation \ + --download ``` ### Citation Format ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/format_verification/configs/citation_format.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/format_verification_citation/ \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resources-server format_verification/citation_format \ + --output-dir data/format_verification_citation/ \ + --mode train_preparation \ + --download ``` ## Testing ```bash -ng_test +entrypoint=resources_servers/format_verification +gym env test --resources-server format_verification ``` ## Licensing diff --git a/resources_servers/frontierscience_judge/README.md b/resources_servers/frontierscience_judge/README.md index abc5405822..890c19b1c3 100644 --- a/resources_servers/frontierscience_judge/README.md +++ b/resources_servers/frontierscience_judge/README.md @@ -35,16 +35,16 @@ judge's full text), and in research mode `rubric_score` plus ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/frontierscience_judge/configs/frontierscience_judge.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server frontierscience_judge # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=frontierscience_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/frontierscience_judge/data/example.jsonl \ - +output_jsonl_fpath=results/frontierscience_judge_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent frontierscience_judge_simple_agent \ + --input resources_servers/frontierscience_judge/data/example.jsonl \ + --output results/frontierscience_judge_rollouts.jsonl \ + --num-repeats 1 ``` The default `judge_model` config points at the public NVIDIA inference diff --git a/resources_servers/gdpval/README.md b/resources_servers/gdpval/README.md index a82cd07ecd..0295fcd26d 100644 --- a/resources_servers/gdpval/README.md +++ b/resources_servers/gdpval/README.md @@ -13,10 +13,11 @@ Two modes via `reward_mode` config: Canonical entry point is the benchmark at `benchmarks/gdpval/`: ```bash -ng_prepare_benchmark "+config_paths=[benchmarks/gdpval/config.yaml]" -ng_e2e_collect_rollouts \ - "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/gdpval/config.yaml]" \ - ++split=benchmark +gym eval prepare --benchmark gdpval +gym eval run \ + --model-type vllm_model \ + --benchmark gdpval \ + --split benchmark ``` See `benchmarks/gdpval/README.md` for the full run recipe. diff --git a/resources_servers/google_search/README.md b/resources_servers/google_search/README.md index 005fea5343..b435edf227 100644 --- a/resources_servers/google_search/README.md +++ b/resources_servers/google_search/README.md @@ -98,14 +98,15 @@ tools=[ ### Running the Server ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/google_search/configs/google_search.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --resources-server google_search -ng_collect_rollouts +agent_name=simple_agent \ - +input_jsonl_fpath=data/Nemotron-RL-knowledge-web_search-mcqa.jsonl \ - +output_jsonl_fpath=results/Nemotron-RL-knowledge-web_search-mcqa.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent simple_agent \ + --input data/Nemotron-RL-knowledge-web_search-mcqa.jsonl \ + --output results/Nemotron-RL-knowledge-web_search-mcqa.jsonl \ + --limit 1 ``` ## Samples diff --git a/resources_servers/gpqa_diamond/README.md b/resources_servers/gpqa_diamond/README.md index b039946429..55dda32ce8 100644 --- a/resources_servers/gpqa_diamond/README.md +++ b/resources_servers/gpqa_diamond/README.md @@ -1,116 +1,117 @@ -# GPQA-Diamond Resources Server - -## Overview - -This resources server evaluates GPQA-Diamond multiple-choice responses with a -GPQA-specific verifier built on top of `resources_servers/mcqa`. - -- Task type: single-turn MCQ -- Domain: `knowledge` -- Dataset prompt format: final line `Answer: LETTER` -- Also accepted by the verifier: `\boxed{X}` and custom regex extraction via - `template_metadata.output_regex` -- Grading mode: `strict_single_letter_boxed` - -## Server Composition - -Use GPQA-Diamond with: - -- `responses_api_agents/simple_agent` -- `responses_api_models/*` (typically `policy_model`) -- `resources_servers/gpqa_diamond` - -The server verifies the model response and returns reward `1.0` for exact -letter match against `expected_answer`, else `0.0`. - -Answer extraction priority is: - -1. `template_metadata.output_regex` if present -2. GPQA fallback parsing of the final answer from `\boxed{...}` or `Answer: X` - -## Dataset Format - -Each JSONL row follows the MCQA request schema: - -- `responses_create_params.input[0].content`: user prompt containing question + options -- `options`: list of letter-to-text maps, e.g. `[{"A": "..."}, {"B": "..."}]` -- `expected_answer`: one of `A/B/C/D` -- `grading_mode`: `strict_single_letter_boxed` -- `template_metadata`: optional per-row metadata, including optional `output_regex` -- `metadata`: passthrough metadata (`explanation`, `subset_for_metrics`, `difficulty`) -- `uuid`: unique row id - -See `data/example.jsonl` for concrete examples. - -Notes: - -- The generated GPQA prompt asks the model to finish with `Answer: LETTER`. -- Although dataset rows keep `grading_mode: strict_single_letter_boxed` for - compatibility with the shared MCQA schema, the GPQA server's custom fallback - parser accepts both `Answer: X` and boxed final answers. - -## Preprocessing Raw GPQA-Diamond - -Full train data is not stored in this repo. - -`dataset_preprocess.py` downloads the GPQA Diamond split from -`Idavidrein/gpqa` on HuggingFace, writes a normalized raw dump, and then writes -the Gym-formatted training set into `data/`. - -From the repository root: - -```bash -python3 resources_servers/gpqa_diamond/dataset_preprocess.py -``` - -This generates: - -- `resources_servers/gpqa_diamond/data/diamond_raw.jsonl` -- `resources_servers/gpqa_diamond/data/train.jsonl` - -`data/example.jsonl` is a curated repo artifact and is not modified by the -preprocess script. There is currently no `validation.jsonl` for this resources -server. - -## Example Usage - -Using a local Nemotron 3 model with `local_vllm_model`: - -```bash -config_paths="responses_api_agents/simple_agent/configs/simple_agent.yaml,responses_api_models/local_vllm_model/configs/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml,resources_servers/gpqa_diamond/configs/gpqa_diamond.yaml" -ng_run "+config_paths=[${config_paths}]" \ - '++policy_model=${inherit_from:NVIDIA-Nemotron-3-Nano-30B-A3B-BF16}' \ - +simple_agent.responses_api_agents.simple_agent.resources_server.name=gpqa_diamond \ - "++NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.responses_api_models.local_vllm_model.vllm_serve_kwargs.mamba_ssm_cache_dtype=float32" \ - "++NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.responses_api_models.local_vllm_model.vllm_serve_kwargs.enable_prefix_caching=False" -``` - -Generic example with `openai_model`: - -```bash -config_paths="responses_api_agents/simple_agent/configs/simple_agent.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/gpqa_diamond/configs/gpqa_diamond.yaml" - -ng_run "+config_paths=[$config_paths]" \ - +simple_agent.responses_api_agents.simple_agent.resources_server.name=gpqa_diamond - -ng_collect_rollouts \ - +agent_name=simple_agent \ - +input_jsonl_fpath=resources_servers/gpqa_diamond/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/gpqa_diamond/data/example_rollouts.jsonl \ - +limit=3 -``` - -`ng_collect_rollouts` also writes sidecar files next to `output_jsonl_fpath`, matching -the same pattern as `test_rollouts*`: - -- `*_materialized_inputs.jsonl` -- `*_reward_profiling.jsonl` -- `*_agent_metrics.json` - -## Licensing - -Code: Apache 2.0 -Configured train dataset license metadata: MIT - +# GPQA-Diamond Resources Server + +## Overview + +This resources server evaluates GPQA-Diamond multiple-choice responses with a +GPQA-specific verifier built on top of `resources_servers/mcqa`. + +- Task type: single-turn MCQ +- Domain: `knowledge` +- Dataset prompt format: final line `Answer: LETTER` +- Also accepted by the verifier: `\boxed{X}` and custom regex extraction via + `template_metadata.output_regex` +- Grading mode: `strict_single_letter_boxed` + +## Server Composition + +Use GPQA-Diamond with: + +- `responses_api_agents/simple_agent` +- `responses_api_models/*` (typically `policy_model`) +- `resources_servers/gpqa_diamond` + +The server verifies the model response and returns reward `1.0` for exact +letter match against `expected_answer`, else `0.0`. + +Answer extraction priority is: + +1. `template_metadata.output_regex` if present +2. GPQA fallback parsing of the final answer from `\boxed{...}` or `Answer: X` + +## Dataset Format + +Each JSONL row follows the MCQA request schema: + +- `responses_create_params.input[0].content`: user prompt containing question + options +- `options`: list of letter-to-text maps, e.g. `[{"A": "..."}, {"B": "..."}]` +- `expected_answer`: one of `A/B/C/D` +- `grading_mode`: `strict_single_letter_boxed` +- `template_metadata`: optional per-row metadata, including optional `output_regex` +- `metadata`: passthrough metadata (`explanation`, `subset_for_metrics`, `difficulty`) +- `uuid`: unique row id + +See `data/example.jsonl` for concrete examples. + +Notes: + +- The generated GPQA prompt asks the model to finish with `Answer: LETTER`. +- Although dataset rows keep `grading_mode: strict_single_letter_boxed` for + compatibility with the shared MCQA schema, the GPQA server's custom fallback + parser accepts both `Answer: X` and boxed final answers. + +## Preprocessing Raw GPQA-Diamond + +Full train data is not stored in this repo. + +`dataset_preprocess.py` downloads the GPQA Diamond split from +`Idavidrein/gpqa` on HuggingFace, writes a normalized raw dump, and then writes +the Gym-formatted training set into `data/`. + +From the repository root: + +```bash +python3 resources_servers/gpqa_diamond/dataset_preprocess.py +``` + +This generates: + +- `resources_servers/gpqa_diamond/data/diamond_raw.jsonl` +- `resources_servers/gpqa_diamond/data/train.jsonl` + +`data/example.jsonl` is a curated repo artifact and is not modified by the +preprocess script. There is currently no `validation.jsonl` for this resources +server. + +## Example Usage + +Using a local Nemotron 3 model with `local_vllm_model`: + +```bash +gym env start \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type local_vllm_model/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 \ + --resources-server gpqa_diamond \ + '++policy_model=${inherit_from:NVIDIA-Nemotron-3-Nano-30B-A3B-BF16}' \ + +simple_agent.responses_api_agents.simple_agent.resources_server.name=gpqa_diamond \ + ++NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.responses_api_models.local_vllm_model.vllm_serve_kwargs.mamba_ssm_cache_dtype=float32 \ + ++NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.responses_api_models.local_vllm_model.vllm_serve_kwargs.enable_prefix_caching=False +``` + +Generic example with `openai_model`: + +```bash +gym env start \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type openai_model \ + --resources-server gpqa_diamond \ + +simple_agent.responses_api_agents.simple_agent.resources_server.name=gpqa_diamond + +gym eval run --no-serve \ + --agent simple_agent \ + --input resources_servers/gpqa_diamond/data/example.jsonl \ + --output resources_servers/gpqa_diamond/data/example_rollouts.jsonl \ + --limit 3 +``` + +`gym eval run` also writes sidecar files next to the `--output` file, matching +the same pattern as `test_rollouts*`: + +- `*_materialized_inputs.jsonl` +- `*_reward_profiling.jsonl` +- `*_agent_metrics.json` + +## Licensing + +Code: Apache 2.0 +Configured train dataset license metadata: MIT + diff --git a/resources_servers/graphwalks/README.md b/resources_servers/graphwalks/README.md index 5f11f4de55..cce3740a62 100644 --- a/resources_servers/graphwalks/README.md +++ b/resources_servers/graphwalks/README.md @@ -25,16 +25,18 @@ https://github.com/NVIDIA-NeMo/Skills/blob/main/nemo_skills/evaluation/evaluator ## Start environment ```bash -ng_run "+config_paths=[resources_servers/graphwalks/configs/graphwalks.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --resources-server graphwalks \ + --model-type vllm_model ``` ## Collect example rollouts ```bash -ng_collect_rollouts \ - +agent_name=graphwalks_simple_agent \ - +input_jsonl_fpath=resources_servers/graphwalks/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/graphwalks/data/example_rollouts.jsonl +gym eval run --no-serve \ + --agent graphwalks_simple_agent \ + --input resources_servers/graphwalks/data/example.jsonl \ + --output resources_servers/graphwalks/data/example_rollouts.jsonl ``` For the full benchmark run see diff --git a/resources_servers/grl_sokoban/README.md b/resources_servers/grl_sokoban/README.md index b5ea29628e..efd1bd6036 100644 --- a/resources_servers/grl_sokoban/README.md +++ b/resources_servers/grl_sokoban/README.md @@ -11,19 +11,20 @@ Single-box Sokoban puzzle environment. The environment is implemented under `res Spin up the server alongside a compatible agent: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -responses_api_agents/gymnasium_agent/configs/gymnasium_agent.yaml,\ -resources_servers/grl_sokoban/configs/grl_sokoban.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --config responses_api_agents/gymnasium_agent/configs/gymnasium_agent.yaml \ + --resources-server grl_sokoban ``` Collect trajectories: ```bash -ng_collect_rollouts +agent_name=grl_sokoban_gymnasium_agent \ - +input_jsonl_fpath=resources_servers/grl_sokoban/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/grl_sokoban/data/example_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent grl_sokoban_gymnasium_agent \ + --input resources_servers/grl_sokoban/data/example.jsonl \ + --output resources_servers/grl_sokoban/data/example_rollouts.jsonl \ + --limit 5 ``` ## Data diff --git a/resources_servers/grl_tetris/README.md b/resources_servers/grl_tetris/README.md index a8868397ef..50792bb867 100644 --- a/resources_servers/grl_tetris/README.md +++ b/resources_servers/grl_tetris/README.md @@ -12,18 +12,19 @@ GRL Tetris environment in Gymnasium style. The model emits one or more ` Start NeMo Gym servers ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -responses_api_agents/gymnasium_agent/configs/gymnasium_agent.yaml,\ -resources_servers/grl_tetris/configs/grl_tetris.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --config responses_api_agents/gymnasium_agent/configs/gymnasium_agent.yaml \ + --resources-server grl_tetris ``` Collect trajectories: ```bash -ng_collect_rollouts +agent_name=grl_tetris_gymnasium_agent \ - +input_jsonl_fpath=resources_servers/grl_tetris/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/grl_tetris/data/example_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent grl_tetris_gymnasium_agent \ + --input resources_servers/grl_tetris/data/example.jsonl \ + --output resources_servers/grl_tetris/data/example_rollouts.jsonl \ + --limit 5 ``` diff --git a/resources_servers/hotpotqa_qa/README.md b/resources_servers/hotpotqa_qa/README.md index 7ef8d91d00..2f3224e13c 100644 --- a/resources_servers/hotpotqa_qa/README.md +++ b/resources_servers/hotpotqa_qa/README.md @@ -30,19 +30,19 @@ pass@k for each channel. ## Running servers ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/hotpotqa_qa/configs/hotpotqa_qa.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --resources-server hotpotqa_qa ``` ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=hotpotqa_qa_simple_agent \ - +input_jsonl_fpath=resources_servers/hotpotqa_qa/data/example.jsonl \ - +output_jsonl_fpath=results/hotpotqa_qa_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent hotpotqa_qa_simple_agent \ + --input resources_servers/hotpotqa_qa/data/example.jsonl \ + --output results/hotpotqa_qa_rollouts.jsonl \ + --num-repeats 1 ``` # Licensing information diff --git a/resources_servers/ifbench/README.md b/resources_servers/ifbench/README.md index db95933b68..f2e4a48b0e 100644 --- a/resources_servers/ifbench/README.md +++ b/resources_servers/ifbench/README.md @@ -45,17 +45,18 @@ This clones the repo, applies patches, and writes the `.installed` marker. On su ## Example Usage ```bash -config_paths="benchmarks/ifbench/config.yaml,responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[$config_paths]" - -ng_collect_rollouts \ - +agent_name=ifbench_benchmark_simple_agent \ - +input_jsonl_fpath=resources_servers/ifbench/data/example.jsonl \ - +prompt_config=benchmarks/ifbench/prompts/default.yaml \ - +output_jsonl_fpath=/tmp/ifbench_rollouts.jsonl \ - +num_repeats=1 \ - "+responses_create_params={max_output_tokens: 4096, temperature: 0.0}" +gym env start \ + --benchmark ifbench \ + --model-type openai_model + +gym eval run --no-serve \ + --agent ifbench_benchmark_simple_agent \ + --input resources_servers/ifbench/data/example.jsonl \ + --output /tmp/ifbench_rollouts.jsonl \ + --num-repeats 1 \ + --max-output-tokens 4096 \ + --temperature 0.0 \ + --prompt-config benchmarks/ifbench/prompts/default.yaml ``` ## Licensing diff --git a/resources_servers/ifbench/setup_ifbench.py b/resources_servers/ifbench/setup_ifbench.py index b610d8e11d..5e9948b372 100644 --- a/resources_servers/ifbench/setup_ifbench.py +++ b/resources_servers/ifbench/setup_ifbench.py @@ -20,7 +20,7 @@ This way, we can just `import instructions_registry`. IFbench dependencies (spaCy, nltk, syllapy, etc.) are listed in requirements.txt -and installed during `ng_run`. The `.installed` marker tracks whether the git +and installed during `gym env start`. The `.installed` marker tracks whether the git clone has been done. """ diff --git a/resources_servers/imo_gradingbench/README.md b/resources_servers/imo_gradingbench/README.md index 18a5e910db..93013666a8 100644 --- a/resources_servers/imo_gradingbench/README.md +++ b/resources_servers/imo_gradingbench/README.md @@ -68,14 +68,14 @@ grade. ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/imo_gradingbench/configs/imo_gradingbench.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server imo_gradingbench # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=imo_gradingbench_simple_agent \ - +input_jsonl_fpath=resources_servers/imo_gradingbench/data/example.jsonl \ - +output_jsonl_fpath=results/imo_gradingbench_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent imo_gradingbench_simple_agent \ + --input resources_servers/imo_gradingbench/data/example.jsonl \ + --output results/imo_gradingbench_rollouts.jsonl \ + --num-repeats 1 ``` diff --git a/resources_servers/imo_proofbench_judge/README.md b/resources_servers/imo_proofbench_judge/README.md index 37ee65a27a..3d2db3b890 100644 --- a/resources_servers/imo_proofbench_judge/README.md +++ b/resources_servers/imo_proofbench_judge/README.md @@ -22,9 +22,9 @@ Use ``benchmarks/imo_proofbench/`` for the IMO-ProofBench dataset. ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/imo_proofbench_judge/configs/imo_proofbench_judge.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --model-type vllm_model \ + --resources-server imo_proofbench_judge \ +judge_base_url=https://generativelanguage.googleapis.com/v1beta/openai \ "+judge_api_key=$GEMINI_API_KEY" \ +judge_model_name=gemini-2.5-pro @@ -33,11 +33,11 @@ ng_run "+config_paths=[$config_paths]" \ ## Collecting rollouts (5-example smoke test) ```bash -ng_collect_rollouts \ - +agent_name=imo_proofbench_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/imo_proofbench_judge/data/example.jsonl \ - +output_jsonl_fpath=results/imo_proofbench_judge_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent imo_proofbench_judge_simple_agent \ + --input resources_servers/imo_proofbench_judge/data/example.jsonl \ + --output results/imo_proofbench_judge_rollouts.jsonl \ + --num-repeats 1 ``` ## Notes diff --git a/resources_servers/instruction_following/README.md b/resources_servers/instruction_following/README.md index c47f051324..7e184b5773 100644 --- a/resources_servers/instruction_following/README.md +++ b/resources_servers/instruction_following/README.md @@ -47,21 +47,23 @@ The dataset can be found at https://huggingface.co/datasets/nvidia/Nemotron-RL-i ### Usage ```bash -config_paths="responses_api_agents/simple_agent/configs/simple_agent.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/instruction_following/configs/instruction_following.yaml" -ng_run "+config_paths=[$config_paths]" "+simple_agent.responses_api_agents.simple_agent.resources_server.name=instruction_following" - -ng_download_dataset_from_gitlab \ - +dataset_name=instruction_following \ - +version=0.0.1 \ - +artifact_fpath=instruction_following.jsonl \ - +output_fpath=data/instruction_following.jsonl - -ng_collect_rollouts \ - +agent_name=simple_agent \ - +input_jsonl_fpath=data/instruction_following.jsonl \ - +output_jsonl_fpath=data/instruction_following_output.jsonl +limit=5 +gym env start \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type openai_model \ + --resources-server instruction_following \ + +simple_agent.responses_api_agents.simple_agent.resources_server.name=instruction_following + +gym dataset download --storage gitlab \ + --name instruction_following \ + --revision 0.0.1 \ + --artifact instruction_following.jsonl \ + --output data/instruction_following.jsonl + +gym eval run --no-serve \ + --agent simple_agent \ + --input data/instruction_following.jsonl \ + --output data/instruction_following_output.jsonl \ + --limit 5 ``` ### Rollout example @@ -81,7 +83,7 @@ We run reward profiling with Qwen/Qwen3-30B-A3B-Instruct-2507. We evaluate the m ## Testing ```bash -ng_test +entrypoint=resources_servers/instruction_following/ +gym env test --resources-server instruction_following ``` diff --git a/resources_servers/inverse_if/README.md b/resources_servers/inverse_if/README.md index 3bcdf63bcd..64cf4a1f6e 100644 --- a/resources_servers/inverse_if/README.md +++ b/resources_servers/inverse_if/README.md @@ -6,17 +6,18 @@ Evaluates model responses on the **Inverse IF** (Instruction Following) benchmar ```bash # 1. Run unit tests -ng_test +entrypoint=resources_servers/inverse_if +gym env test --resources-server inverse_if # 2. Start servers (in terminal 1) -config_paths="resources_servers/inverse_if/configs/inverse_if.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --resources-server inverse_if \ + --model-type vllm_model # 3. Collect rollouts on example data (in terminal 2) -ng_collect_rollouts \ - +agent_name=inverse_if_simple_agent \ - +input_jsonl_fpath=resources_servers/inverse_if/data/example.jsonl \ - +output_jsonl_fpath=/tmp/inverse_if_rollouts.jsonl +gym eval run --no-serve \ + --agent inverse_if_simple_agent \ + --input resources_servers/inverse_if/data/example.jsonl \ + --output /tmp/inverse_if_rollouts.jsonl ``` ## Overview @@ -58,10 +59,10 @@ The environment: The `data/example.jsonl` file contains 3 synthetic tasks ready to use: ```bash -ng_collect_rollouts \ - +agent_name=inverse_if_simple_agent \ - +input_jsonl_fpath=resources_servers/inverse_if/data/example.jsonl \ - +output_jsonl_fpath=/tmp/test_rollouts.jsonl +gym eval run --no-serve \ + --agent inverse_if_simple_agent \ + --input resources_servers/inverse_if/data/example.jsonl \ + --output /tmp/test_rollouts.jsonl ``` ### Option B: Full Dataset Setup @@ -88,10 +89,10 @@ ng_collect_rollouts \ 2. **Run on full dataset**: ```bash - ng_collect_rollouts \ - +agent_name=inverse_if_simple_agent \ - +input_jsonl_fpath=resources_servers/inverse_if/data/inverse_if.jsonl \ - +output_jsonl_fpath=/tmp/inverse_if_rollouts.jsonl + gym eval run --no-serve \ + --agent inverse_if_simple_agent \ + --input resources_servers/inverse_if/data/inverse_if.jsonl \ + --output /tmp/inverse_if_rollouts.jsonl ``` ## Testing @@ -100,7 +101,7 @@ ng_collect_rollouts \ ```bash # Run all unit tests -ng_test +entrypoint=resources_servers/inverse_if +gym env test --resources-server inverse_if # Or run directly with pytest cd resources_servers/inverse_if @@ -117,17 +118,18 @@ Tests cover: 1. **Start servers**: ```bash - config_paths="resources_servers/inverse_if/configs/inverse_if.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml" - ng_run "+config_paths=[${config_paths}]" + gym env start \ + --resources-server inverse_if \ + --model-type vllm_model ``` 2. **In another terminal, run on example data**: ```bash - ng_collect_rollouts \ - +agent_name=inverse_if_simple_agent \ - +input_jsonl_fpath=resources_servers/inverse_if/data/example.jsonl \ - +output_jsonl_fpath=/tmp/inverse_if_rollouts.jsonl \ - +limit=3 + gym eval run --no-serve \ + --agent inverse_if_simple_agent \ + --input resources_servers/inverse_if/data/example.jsonl \ + --output /tmp/inverse_if_rollouts.jsonl \ + --limit 3 ``` 3. **View results**: @@ -227,7 +229,7 @@ Each line contains: Key transformations: - Inconsistent rubric keys normalised to `{"id", "criteria"}` - Per-task judge template and system prompt extracted from messages -- `responses_create_params` wraps input for `ng_collect_rollouts` +- `responses_create_params` wraps input for `gym eval run --no-serve` - `model_responses` are discarded ## File Structure diff --git a/resources_servers/inverse_if/data/README.md b/resources_servers/inverse_if/data/README.md index 4e12d4be01..f3c5839142 100644 --- a/resources_servers/inverse_if/data/README.md +++ b/resources_servers/inverse_if/data/README.md @@ -35,10 +35,10 @@ The `example.jsonl` file contains 3 synthetic tasks for quick testing: **Usage:** ```bash -ng_collect_rollouts \ - +agent_name=inverse_if_simple_agent \ - +input_jsonl_fpath=resources_servers/inverse_if/data/example.jsonl \ - +output_jsonl_fpath=/tmp/test_rollouts.jsonl +gym eval run --no-serve \ + --agent inverse_if_simple_agent \ + --input resources_servers/inverse_if/data/example.jsonl \ + --output /tmp/test_rollouts.jsonl ``` ## Raw JSON Format @@ -89,7 +89,7 @@ Each line in the JSONL file: **Key transformations:** - Inconsistent rubric keys normalised to `{"id", "criteria"}` - Per-task judge template and system prompt extracted from messages -- `responses_create_params` wrapper required by `ng_collect_rollouts` +- `responses_create_params` wrapper required by `gym eval run --no-serve` - `model_responses` discarded ## Regenerating JSONL Files diff --git a/resources_servers/inverse_if/dataset_preprocess.py b/resources_servers/inverse_if/dataset_preprocess.py index bd510ef956..1c56ff65c7 100644 --- a/resources_servers/inverse_if/dataset_preprocess.py +++ b/resources_servers/inverse_if/dataset_preprocess.py @@ -27,7 +27,7 @@ - Normalises inconsistent rubric key names (criteria, criteria1, rule, question, …) into a uniform ``criteria`` field - Falls back to parsing ``response_reference`` when the ``rubrics`` array is empty -- Wraps the prompt in ``responses_create_params`` for ``ng_collect_rollouts`` +- Wraps the prompt in ``responses_create_params`` for ``gym eval run`` """ import argparse @@ -193,7 +193,7 @@ def process_task(task: dict, fallback_id: str = "unknown") -> dict[str, Any]: "type": "responses_api_agents", "name": "inverse_if_simple_agent", }, - # Input wrapped for ng_collect_rollouts + # Input wrapped for gym eval run "responses_create_params": { "input": [{"role": "user", "content": prompt}], }, diff --git a/resources_servers/jailbreak_detection/README.md b/resources_servers/jailbreak_detection/README.md index 04acbc7f8e..122dee20bd 100644 --- a/resources_servers/jailbreak_detection/README.md +++ b/resources_servers/jailbreak_detection/README.md @@ -27,16 +27,21 @@ The example dataset includes various jailbreak attack patterns: 1. Start the servers: ```bash -ng_run "+config_paths=[resources_servers/jailbreak_detection/configs/jailbreak_detection_nemotron_combined_reward_tp8.yaml,responses_api_models/openai_model/configs/openai_model.yaml,resources_servers/jailbreak_detection/configs/safety_judge_model.yaml]" +gym env start \ + --resources-server jailbreak_detection/jailbreak_detection_nemotron_combined_reward_tp8 \ + --model-type openai_model \ + --resources-server jailbreak_detection/safety_judge_model ``` 2. Collect rollouts: ```bash -ng_collect_rollouts \ - "+config_paths=[resources_servers/jailbreak_detection/configs/jailbreak_detection_nemotron_combined_reward_tp8.yaml,responses_api_models/openai_model/configs/openai_model.yaml,resources_servers/jailbreak_detection/configs/safety_judge_model.yaml]" \ - +agent_name=jailbreak_detection_simple_agent \ - +input_jsonl_fpath=resources_servers/jailbreak_detection/data/example.jsonl \ - +output_jsonl_fpath=results/jailbreak_detection_rollouts.jsonl +gym eval run --no-serve \ + --resources-server jailbreak_detection/jailbreak_detection_nemotron_combined_reward_tp8 \ + --model-type openai_model \ + --resources-server jailbreak_detection/safety_judge_model \ + --agent jailbreak_detection_simple_agent \ + --input resources_servers/jailbreak_detection/data/example.jsonl \ + --output results/jailbreak_detection_rollouts.jsonl ``` ## Configuration diff --git a/resources_servers/labbench2_vlm/README.md b/resources_servers/labbench2_vlm/README.md index 426a8d4cf8..2eddbab829 100644 --- a/resources_servers/labbench2_vlm/README.md +++ b/resources_servers/labbench2_vlm/README.md @@ -128,7 +128,7 @@ the highest-k values for `pass@1[avg-of-{k}]/accuracy` and `pass@{k}/accuracy`. Media files (scientific figures and table images/PDFs) are downloaded from a public GCS bucket by `prepare_data.py`. Validation JSONL can also be downloaded via `gitlab_identifier` (for internal NVIDIA users) using -`ng_prepare_data +should_download=true +data_source=gitlab`. +`gym dataset collate +should_download=true +data_source=gitlab`. Media files must always be downloaded separately via `prepare_data.py` -- the `gitlab_identifier` mechanism only handles the lightweight JSONL files. @@ -178,22 +178,27 @@ putting them in `env.yaml`. Start the servers: ```bash -ng_run "+config_paths=[resources_servers/labbench2_vlm/configs/labbench2_vlm.yaml,resources_servers/labbench2_vlm/configs/judge_model_openai.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env start \ + --resources-server labbench2_vlm \ + --resources-server labbench2_vlm/judge_model_openai \ + --model-type openai_model ``` Collect rollouts: ```bash -ng_collect_rollouts \ - +agent_name=labbench2_vlm_simple_agent \ - "+config_paths=[resources_servers/labbench2_vlm/configs/labbench2_vlm.yaml,resources_servers/labbench2_vlm/configs/judge_model_openai.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - +input_jsonl_fpath=resources_servers/labbench2_vlm/data/figqa2_img_validation.jsonl \ - +output_jsonl_fpath=results/figqa2_img_rollouts.jsonl \ - +num_repeats=1 \ +gym eval run --no-serve \ + --resources-server labbench2_vlm \ + --resources-server labbench2_vlm/judge_model_openai \ + --model-type openai_model \ + --agent labbench2_vlm_simple_agent \ + --input resources_servers/labbench2_vlm/data/figqa2_img_validation.jsonl \ + --output results/figqa2_img_rollouts.jsonl \ + --num-repeats 1 \ "+responses_create_params={max_output_tokens: 2048, reasoning: {effort: high}}" ``` -`ng_collect_rollouts` writes sidecar files next to `output_jsonl_fpath`: +`gym eval run --no-serve` writes sidecar files next to the `--output` file: - `*_materialized_inputs.jsonl` - `*_aggregate_metrics.json` @@ -202,12 +207,14 @@ Run the example data for a quick smoke test (works immediately after `git clone` because `data/test_media/` and `data/example.jsonl` are committed): ```bash -ng_collect_rollouts \ - +agent_name=labbench2_vlm_simple_agent \ - "+config_paths=[resources_servers/labbench2_vlm/configs/labbench2_vlm.yaml,resources_servers/labbench2_vlm/configs/judge_model_openai.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - +input_jsonl_fpath=resources_servers/labbench2_vlm/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/labbench2_vlm/data/example_rollouts.jsonl \ - +num_repeats=1 \ +gym eval run --no-serve \ + --resources-server labbench2_vlm \ + --resources-server labbench2_vlm/judge_model_openai \ + --model-type openai_model \ + --agent labbench2_vlm_simple_agent \ + --input resources_servers/labbench2_vlm/data/example.jsonl \ + --output resources_servers/labbench2_vlm/data/example_rollouts.jsonl \ + --num-repeats 1 \ "+responses_create_params={max_output_tokens: 2048, reasoning: {effort: high}}" ``` @@ -216,20 +223,22 @@ rollout output). Regenerate it locally with the command above. ### One-shot alternative -`ng_e2e_collect_rollouts` starts the server stack, preprocesses, and collects -rollouts in a single command (don't run `ng_run` separately). Input path and +`gym eval run` starts the server stack, preprocesses, and collects +rollouts in a single command (don't run `gym env start` separately). Input path and agent ref are auto-derived from the dataset entry in the chained config -(`++split` picks which one): +(`--split` picks which one): ```bash -ng_e2e_collect_rollouts \ - "+config_paths=[resources_servers/labbench2_vlm/configs/labbench2_vlm.yaml,resources_servers/labbench2_vlm/configs/judge_model_openai.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - ++split=validation \ - ++output_jsonl_fpath=results/labbench2_vlm_validation.jsonl \ - +num_samples_in_parallel=16 +gym eval run \ + --resources-server labbench2_vlm \ + --resources-server labbench2_vlm/judge_model_openai \ + --model-type openai_model \ + --split validation \ + --output results/labbench2_vlm_validation.jsonl \ + --concurrency 16 ``` -For a fast smoke test, add `+limit=10 +num_repeats=1`. +For a fast smoke test, add `--limit 10 --num-repeats 1`. ## Throttling @@ -238,10 +247,10 @@ endpoints see roughly `2 × num_samples_in_parallel` concurrent requests. On a hosted endpoint you'll likely hit rate limits or socket errors (`Hit N global ClientOSError`) well before saturating your machine. -Cap concurrency with `+num_samples_in_parallel=`: +Cap concurrency with `--concurrency `: ```bash -ng_collect_rollouts ... +num_samples_in_parallel=16 +gym eval run --no-serve ... --concurrency 16 ``` Start around 16 and bump up if it holds. diff --git a/resources_servers/longmt_eval/README.md b/resources_servers/longmt_eval/README.md index de5584b38e..69f4f3c7e7 100644 --- a/resources_servers/longmt_eval/README.md +++ b/resources_servers/longmt_eval/README.md @@ -145,9 +145,9 @@ and output rows. ### Reward profiling from pre-baked rollouts (no GPU, no model server) ```bash -ng_reward_profile \ - +materialized_inputs_jsonl_fpath=resources_servers/longmt_eval/data/example_rollouts_materialized_inputs.jsonl \ - +rollouts_jsonl_fpath=resources_servers/longmt_eval/data/example_rollouts.jsonl +gym eval profile \ + --inputs resources_servers/longmt_eval/data/example_rollouts_materialized_inputs.jsonl \ + --rollouts resources_servers/longmt_eval/data/example_rollouts.jsonl ``` Outputs `example_rollouts_reward_profiling.jsonl` (per-task stats) and @@ -158,20 +158,22 @@ rollouts file. ```bash # Start servers (smoke-test mode — no GPU needed for the verifier) -ng_run "+config_paths=[resources_servers/longmt_eval/configs/longmt_eval.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" \ - "++longmt_eval.resources_servers.longmt_eval.compute_segale=false" & +gym env start \ + --resources-server longmt_eval \ + --model-type vllm_model & \ + ++longmt_eval.resources_servers.longmt_eval.compute_segale=false # Collect rollouts — also writes results/longmt_eval_rollouts_materialized_inputs.jsonl -ng_collect_rollouts \ - +agent_name=longmt_eval_simple_agent \ - +input_jsonl_fpath=resources_servers/longmt_eval/data/example.jsonl \ - +output_jsonl_fpath=results/longmt_eval_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent longmt_eval_simple_agent \ + --input resources_servers/longmt_eval/data/example.jsonl \ + --output results/longmt_eval_rollouts.jsonl \ + --num-repeats 1 # Profile rewards from the collected rollouts -ng_reward_profile \ - +materialized_inputs_jsonl_fpath=results/longmt_eval_rollouts_materialized_inputs.jsonl \ - +rollouts_jsonl_fpath=results/longmt_eval_rollouts.jsonl +gym eval profile \ + --inputs results/longmt_eval_rollouts_materialized_inputs.jsonl \ + --rollouts results/longmt_eval_rollouts.jsonl ``` For a full SLURM run with SEGALE enabled on WMT24++ see diff --git a/resources_servers/math_advanced_calculations/README.md b/resources_servers/math_advanced_calculations/README.md index 2aa2601e90..3a5c7c90a4 100644 --- a/resources_servers/math_advanced_calculations/README.md +++ b/resources_servers/math_advanced_calculations/README.md @@ -8,18 +8,18 @@ Commands - Spin up server: ``` - config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/math_advanced_calculations/configs/math_advanced_calculations.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --resources-server math_advanced_calculations ``` Collect trajectories: ``` -ng_collect_rollouts +agent_name=math_advanced_calculations_simple_agent \ - +input_jsonl_fpath=resources_servers/math_advanced_calculations/data/train.jsonl \ - +output_jsonl_fpath=results/math_advanced_calculations_trajectory_collection.jso -nl \ - +limit=1 +gym eval run --no-serve \ + --agent math_advanced_calculations_simple_agent \ + --input resources_servers/math_advanced_calculations/data/train.jsonl \ + --output results/math_advanced_calculations_trajectory_collection.jsonl \ + --limit 1 ``` Data links: https://huggingface.co/datasets/nvidia/Nemotron-RL-math-advanced_calculations diff --git a/resources_servers/math_proof_judgement/README.md b/resources_servers/math_proof_judgement/README.md index 3959e44044..3a395eb777 100644 --- a/resources_servers/math_proof_judgement/README.md +++ b/resources_servers/math_proof_judgement/README.md @@ -66,14 +66,14 @@ parser name that matches your model's reasoning tokens (see `vllm serve ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/math_proof_judgement/configs/math_proof_judgement.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server math_proof_judgement # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=math_proof_judgement_simple_agent \ - +input_jsonl_fpath=resources_servers/math_proof_judgement/data/example.jsonl \ - +output_jsonl_fpath=results/math_proof_judgement_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent math_proof_judgement_simple_agent \ + --input resources_servers/math_proof_judgement/data/example.jsonl \ + --output results/math_proof_judgement_rollouts.jsonl \ + --num-repeats 1 ``` diff --git a/resources_servers/math_with_autograder/README.md b/resources_servers/math_with_autograder/README.md index 100c400146..75ad3030a3 100644 --- a/resources_servers/math_with_autograder/README.md +++ b/resources_servers/math_with_autograder/README.md @@ -35,8 +35,10 @@ existing math_with_judge consumers. ## Run servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,resources_servers/math_with_autograder/configs/math_with_autograder.yaml,resources_servers/math_with_autograder/configs/judge_gptoss20b.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server math_with_autograder \ + --resources-server math_with_autograder/judge_gptoss20b ``` The bundled `judge_gptoss20b.yaml` wires the autograder judge to @@ -47,11 +49,11 @@ swap to a different OpenAI-compatible endpoint. ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=math_with_autograder_simple_agent \ - +input_jsonl_fpath=resources_servers/math_with_autograder/data/example.jsonl \ - +output_jsonl_fpath=results/math_with_autograder_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent math_with_autograder_simple_agent \ + --input resources_servers/math_with_autograder/data/example.jsonl \ + --output results/math_with_autograder_rollouts.jsonl \ + --num-repeats 1 ``` ## Metrics diff --git a/resources_servers/math_with_code/README.md b/resources_servers/math_with_code/README.md index b3a43a2a98..85bf7a9472 100644 --- a/resources_servers/math_with_code/README.md +++ b/resources_servers/math_with_code/README.md @@ -95,23 +95,31 @@ OR If you want to download directly ``` -ng_download_dataset_from_gitlab \ - +dataset_name=open_math_reasoning_problems_tool \ - +version=0.0.1 \ - +artifact_fpath=open_math_reasoning_problems_tool.jsonl \ - +output_fpath=data/open_math_reasoning_problems_tool.jsonl +gym dataset download --storage gitlab \ + --name open_math_reasoning_problems_tool \ + --revision 0.0.1 \ + --artifact open_math_reasoning_problems_tool.jsonl \ + --output data/open_math_reasoning_problems_tool.jsonl ``` Start server ``` -ng_run "+config_paths=[responses_api_agents/simple_agent/configs/simple_agent.yaml,responses_api_models/openai_model/configs/openai_model.yaml,resources_servers/math_with_code/configs/math_with_code.yaml]" +simple_agent.responses_api_agents.simple_agent.resources_server.name=math_with_code +gym env start \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type openai_model \ + --resources-server math_with_code \ + +simple_agent.responses_api_agents.simple_agent.resources_server.name=math_with_code ``` Collect trajectories ``` -ng_collect_rollouts +agent_name=simple_agent +input_jsonl_fpath=data/open_math_reasoning_problems_tool.jsonl +output_jsonl_fpath=results/open_math_reasoning_problems_tool_output_new.jsonl +limit=1 +gym eval run --no-serve \ + --agent simple_agent \ + --input data/open_math_reasoning_problems_tool.jsonl \ + --output results/open_math_reasoning_problems_tool_output_new.jsonl \ + --limit 1 ``` diff --git a/resources_servers/math_with_judge/README.md b/resources_servers/math_with_judge/README.md index 0fdfe0318b..97d1cd66ca 100644 --- a/resources_servers/math_with_judge/README.md +++ b/resources_servers/math_with_judge/README.md @@ -11,38 +11,38 @@ on Hugging Face. ## Running servers The following are example commands for running this resources server, along with the simple agent and an OpenAI model: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -resources_servers/math_with_judge/configs/math_with_judge.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --model-type openai_model \ + --resources-server math_with_judge \ +math_with_judge.resources_servers.math_with_judge.judge_model_server.name=policy_model ``` To download the OpenMathReasoning dataset, the following command can be run: ```bash -ng_download_dataset_from_gitlab \ - +dataset_name=math_open_math_reasoning \ - +version=0.0.1 \ - +artifact_fpath=open_math_reasoning_problems.jsonl \ - +output_fpath=data/open_math_reasoning_problems.jsonl +gym dataset download --storage gitlab \ + --name math_open_math_reasoning \ + --revision 0.0.1 \ + --artifact open_math_reasoning_problems.jsonl \ + --output data/open_math_reasoning_problems.jsonl ``` Then, rollouts can be collected using a command such as the following: ```bash -ng_collect_rollouts \ - +agent_name=math_with_judge_simple_agent \ - +input_jsonl_fpath=data/open_math_reasoning_problems.jsonl \ - +output_jsonl_fpath=results/example_open_math_reasoning_verify_responses.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent math_with_judge_simple_agent \ + --input data/open_math_reasoning_problems.jsonl \ + --output results/example_open_math_reasoning_verify_responses.jsonl \ + --limit 5 ``` ## Prepare for trajectory collection ```bash -config_paths="resources_servers/math_with_judge/configs/dapo17k_trajectory_collection.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_prepare_data "+config_paths=[$config_paths]" \ - +output_dirpath=data/dapo17k_trajectory_collection \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config resources_servers/math_with_judge/configs/dapo17k_trajectory_collection.yaml \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/dapo17k_trajectory_collection \ + --mode train_preparation \ + --download ``` # Licensing information diff --git a/resources_servers/mcqa/README.md b/resources_servers/mcqa/README.md index 4fdb3439ef..909bdfaf48 100644 --- a/resources_servers/mcqa/README.md +++ b/resources_servers/mcqa/README.md @@ -111,28 +111,27 @@ For datasets with custom prompt formats, you can optionally use `template_metada **Standard format (with `grading_mode`):** ```bash -config_paths="responses_api_agents/simple_agent/configs/simple_agent.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/mcqa/configs/mcqa.yaml" - - -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type openai_model \ + --resources-server mcqa \ +simple_agent.responses_api_agents.simple_agent.resources_server.name=mcqa -ng_collect_rollouts \ - +agent_name=simple_agent \ - +input_jsonl_fpath=data/MCQA_filtered_decontaminated.jsonl \ - +output_jsonl_fpath=data/MCQA_filtered_decontaminated_samples_rollouts.jsonl +limit=5 +gym eval run --no-serve \ + --agent simple_agent \ + --input data/MCQA_filtered_decontaminated.jsonl \ + --output data/MCQA_filtered_decontaminated_samples_rollouts.jsonl \ + --limit 5 ``` **With template_metadata (custom regex):** ```bash # Using example file with 5 different custom prompt formats -ng_collect_rollouts \ - +agent_name=simple_agent \ - +input_jsonl_fpath=resources_servers/mcqa/data/example_with_template_metadata.jsonl \ - +output_jsonl_fpath=resources_servers/mcqa/data/example_rollouts_with_template_metadata.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent simple_agent \ + --input resources_servers/mcqa/data/example_with_template_metadata.jsonl \ + --output resources_servers/mcqa/data/example_rollouts_with_template_metadata.jsonl \ + --limit 5 ``` Rollout example diff --git a/resources_servers/mrcr/README.md b/resources_servers/mrcr/README.md index d067f11789..38926e346d 100644 --- a/resources_servers/mrcr/README.md +++ b/resources_servers/mrcr/README.md @@ -27,16 +27,18 @@ always fails. ## Start environment ```bash -ng_run "+config_paths=[resources_servers/mrcr/configs/mrcr.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --resources-server mrcr \ + --model-type vllm_model ``` ## Collect example rollouts ```bash -ng_collect_rollouts \ - +agent_name=mrcr_simple_agent \ - +input_jsonl_fpath=resources_servers/mrcr/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/mrcr/data/example_rollouts.jsonl +gym eval run --no-serve \ + --agent mrcr_simple_agent \ + --input resources_servers/mrcr/data/example.jsonl \ + --output resources_servers/mrcr/data/example_rollouts.jsonl ``` For the full benchmark run see diff --git a/resources_servers/multichallenge/README.md b/resources_servers/multichallenge/README.md index 6e877d9421..8a5c93fc8f 100644 --- a/resources_servers/multichallenge/README.md +++ b/resources_servers/multichallenge/README.md @@ -6,17 +6,18 @@ Evaluates model responses on the **MultiChallenge** benchmark using an LLM judge ```bash # 1. Run unit tests -ng_test +entrypoint=resources_servers/multichallenge +gym env test --resources-server multichallenge # 2. Start servers (in terminal 1) -config_paths="resources_servers/multichallenge/configs/multichallenge.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --resources-server multichallenge \ + --model-type vllm_model # 3. Collect rollouts on example data (in terminal 2) -ng_collect_rollouts \ - +agent_name=multichallenge_simple_agent \ - +input_jsonl_fpath=resources_servers/multichallenge/data/example.jsonl \ - +output_jsonl_fpath=/tmp/multichallenge_rollouts.jsonl +gym eval run --no-serve \ + --agent multichallenge_simple_agent \ + --input resources_servers/multichallenge/data/example.jsonl \ + --output /tmp/multichallenge_rollouts.jsonl ``` ## Overview @@ -40,10 +41,10 @@ The `data/example.jsonl` file contains 3 synthetic tasks ready to use: ```bash # No preprocessing needed - just run -ng_collect_rollouts \ - +agent_name=multichallenge_simple_agent \ - +input_jsonl_fpath=resources_servers/multichallenge/data/example.jsonl \ - +output_jsonl_fpath=/tmp/test_rollouts.jsonl +gym eval run --no-serve \ + --agent multichallenge_simple_agent \ + --input resources_servers/multichallenge/data/example.jsonl \ + --output /tmp/test_rollouts.jsonl ``` ### Option B: Full Dataset Setup @@ -76,10 +77,10 @@ ng_collect_rollouts \ 2. **Run on full dataset**: ```bash - ng_collect_rollouts \ - +agent_name=multichallenge_simple_agent \ - +input_jsonl_fpath=resources_servers/multichallenge/data/advanced.jsonl \ - +output_jsonl_fpath=/tmp/advanced_rollouts.jsonl + gym eval run --no-serve \ + --agent multichallenge_simple_agent \ + --input resources_servers/multichallenge/data/advanced.jsonl \ + --output /tmp/advanced_rollouts.jsonl ``` ## Testing @@ -88,7 +89,7 @@ ng_collect_rollouts \ ```bash # Run all unit tests -ng_test +entrypoint=resources_servers/multichallenge +gym env test --resources-server multichallenge # Or run directly with pytest for more detail cd resources_servers/multichallenge @@ -105,17 +106,18 @@ Tests cover: 1. **Start servers**: ```bash - config_paths="resources_servers/multichallenge/configs/multichallenge.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml" - ng_run "+config_paths=[${config_paths}]" + gym env start \ + --resources-server multichallenge \ + --model-type vllm_model ``` 2. **In another terminal, run on example data**: ```bash - ng_collect_rollouts \ - +agent_name=multichallenge_simple_agent \ - +input_jsonl_fpath=resources_servers/multichallenge/data/example.jsonl \ - +output_jsonl_fpath=/tmp/multichallenge_rollouts.jsonl \ - +limit=3 + gym eval run --no-serve \ + --agent multichallenge_simple_agent \ + --input resources_servers/multichallenge/data/example.jsonl \ + --output /tmp/multichallenge_rollouts.jsonl \ + --limit 3 ``` 3. **View results**: @@ -240,7 +242,7 @@ Each line contains: Key transformations: - `thinking` role messages are excluded from input - `context` is pre-formatted for the LLM judge -- `responses_create_params` wraps input for `ng_collect_rollouts` +- `responses_create_params` wraps input for `gym eval run` ## File Structure diff --git a/resources_servers/multichallenge/data/README.md b/resources_servers/multichallenge/data/README.md index d8db78eb6d..9517579b31 100644 --- a/resources_servers/multichallenge/data/README.md +++ b/resources_servers/multichallenge/data/README.md @@ -45,10 +45,10 @@ The `example.jsonl` file contains 3 synthetic tasks for quick testing: **Usage:** ```bash -ng_collect_rollouts \ - +agent_name=multichallenge_simple_agent \ - +input_jsonl_fpath=resources_servers/multichallenge/data/example.jsonl \ - +output_jsonl_fpath=/tmp/test_rollouts.jsonl +gym eval run --no-serve \ + --agent multichallenge_simple_agent \ + --input resources_servers/multichallenge/data/example.jsonl \ + --output /tmp/test_rollouts.jsonl ``` ## Raw JSON Format @@ -106,7 +106,7 @@ Each line in the JSONL file: **Key transformations:** - `thinking` role messages are **excluded** from `responses_create_params.input` - `context` is a pre-formatted string for the LLM judge (also excludes thinking) -- `responses_create_params` wrapper is required by `ng_collect_rollouts` +- `responses_create_params` wrapper is required by `gym eval run` - `metadata` preserves full original data for reference ## Regenerating JSONL Files diff --git a/resources_servers/multichallenge/dataset_preprocess.py b/resources_servers/multichallenge/dataset_preprocess.py index 798848e16b..cdf945f033 100644 --- a/resources_servers/multichallenge/dataset_preprocess.py +++ b/resources_servers/multichallenge/dataset_preprocess.py @@ -101,7 +101,7 @@ def process_task(task: dict, fallback_id: str = "unknown") -> dict[str, Any]: "type": "responses_api_agents", "name": "multichallenge_simple_agent", }, - # Input messages wrapped in responses_create_params (required by ng_collect_rollouts) + # Input messages wrapped in responses_create_params (required by gym eval run) "responses_create_params": { "input": build_input_messages(task), }, diff --git a/resources_servers/newton_bench/README.md b/resources_servers/newton_bench/README.md index 75e6749a72..efcd34edd7 100644 --- a/resources_servers/newton_bench/README.md +++ b/resources_servers/newton_bench/README.md @@ -91,22 +91,23 @@ vllm serve \ ### Launch servers ```bash -config_paths="resources_servers/newton_bench/configs/newton_bench.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --resources-server newton_bench \ + --model-type vllm_model ``` ### Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=newton_bench_simple_agent \ - +input_jsonl_fpath=resources_servers/newton_bench/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/newton_bench/data/example_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent newton_bench_simple_agent \ + --input resources_servers/newton_bench/data/example.jsonl \ + --output resources_servers/newton_bench/data/example_rollouts.jsonl \ + --limit 5 ``` ## Running Tests ```bash -ng_test +entrypoint=resources_servers/newton_bench +gym env test --resources-server newton_bench ``` ## Qwen/Qwen3-VL-8B-Thinking Evaluation Summary diff --git a/resources_servers/newton_bench/client.py b/resources_servers/newton_bench/client.py index 5020af3fef..43628cce24 100644 --- a/resources_servers/newton_bench/client.py +++ b/resources_servers/newton_bench/client.py @@ -24,7 +24,7 @@ 4. Discover the underlying physical law Prerequisites: -- NewtonBench server running via ng_run +- NewtonBench server running via gym env start - NewtonBench repo cloned in project root (for run_experiment to work) Usage: diff --git a/resources_servers/ns_tools/README.md b/resources_servers/ns_tools/README.md index 571b764e92..eac6eff1f0 100644 --- a/resources_servers/ns_tools/README.md +++ b/resources_servers/ns_tools/README.md @@ -8,23 +8,23 @@ It integrates with the NeMo Skills ToolManager to dynamically load and execute t ## Running servers The following are example commands for running this resources server with the simple agent and a vLLM model: ```bash -config_paths="resources_servers/ns_tools/configs/ns_tools.yaml, \ -resources_servers/math_with_judge/configs/math_with_judge.yaml, \ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[$config_paths]" \ - +policy_base_url="http://localhost:8000/v1" \ - +policy_model_name="Qwen/Qwen3-8B" \ +gym env start \ + --resources-server ns_tools \ + --resources-server math_with_judge \ + --model-type vllm_model \ + --model-url http://localhost:8000/v1 \ + --model Qwen/Qwen3-8B \ ++math_with_judge.resources_servers.math_with_judge.should_use_judge=false \ ++math_with_judge.resources_servers.math_with_judge.judge_model_server.name=policy_model ``` Then, rollouts can be collected using a command such as the following: ```bash -ng_collect_rollouts \ - +agent_name=ns_tools_simple_agent \ - +input_jsonl_fpath=resources_servers/ns_tools/data/example.jsonl \ - +output_jsonl_fpath=data/ns_tools_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent ns_tools_simple_agent \ + --input resources_servers/ns_tools/data/example.jsonl \ + --output data/ns_tools_rollouts.jsonl \ + --limit 5 ``` ## Sample data format @@ -78,9 +78,11 @@ python prepare_dataset.py \ To validate the example data and regenerate metrics: ```bash -ng_prepare_data "+config_paths=[resources_servers/ns_tools/configs/ns_tools.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" \ - +output_dirpath=data/ns_tools \ - +mode=example_validation +gym dataset collate \ + --resources-server ns_tools \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --output-dir data/ns_tools \ + --mode example_validation ``` # Licensing information diff --git a/resources_servers/openenv/README.md b/resources_servers/openenv/README.md index d56f9d4026..8253a2e89f 100644 --- a/resources_servers/openenv/README.md +++ b/resources_servers/openenv/README.md @@ -36,23 +36,25 @@ policy_model_name: Running an environment is a **two-step process**: -1. **Start the servers** with `ng_run` — this launches the resource server, agent server, and model server. Wait until you see `All 3 / 3 servers ready!`. -2. **Collect rollouts** with `ng_collect_rollouts` in a **separate terminal** — this sends the JSONL prompts through the agent, which calls the LLM and environment in a loop, and writes trajectories with rewards to an output file. +1. **Start the servers** with `gym env start` — this launches the resources server, agent server, and model server. Wait until you see `All 3 / 3 servers ready!`. +2. **Collect rollouts** with `gym eval run --no-serve` in a **separate terminal** — this sends the JSONL prompts through the agent, which calls the LLM and environment in a loop, and writes trajectories with rewards to an output file. #### Echo ```bash # Terminal 1: Start servers source .venv/bin/activate -ng_run "+config_paths=[resources_servers/openenv/configs/openenv_echo.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env start \ + --resources-server openenv/openenv_echo \ + --model-type openai_model # Terminal 2: Collect rollouts (after "All 3 / 3 servers ready!") source .venv/bin/activate -ng_collect_rollouts \ - +agent_name=openenv_echo_simple_agent \ - +input_jsonl_fpath=resources_servers/openenv/data/echo/example.jsonl \ - +output_jsonl_fpath=output_echo.jsonl \ - +num_samples_in_parallel=5 +gym eval run --no-serve \ + --agent openenv_echo_simple_agent \ + --input resources_servers/openenv/data/echo/example.jsonl \ + --output output_echo.jsonl \ + --concurrency 5 ``` #### Coding @@ -60,15 +62,17 @@ ng_collect_rollouts \ ```bash # Terminal 1: Start servers source .venv/bin/activate -ng_run "+config_paths=[resources_servers/openenv/configs/openenv_coding.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env start \ + --resources-server openenv/openenv_coding \ + --model-type openai_model # Terminal 2: Collect rollouts source .venv/bin/activate -ng_collect_rollouts \ - +agent_name=openenv_coding_simple_agent \ - +input_jsonl_fpath=resources_servers/openenv/data/coding/example.jsonl \ - +output_jsonl_fpath=output_coding.jsonl \ - +num_samples_in_parallel=5 +gym eval run --no-serve \ + --agent openenv_coding_simple_agent \ + --input resources_servers/openenv/data/coding/example.jsonl \ + --output output_coding.jsonl \ + --concurrency 5 ``` #### Maze @@ -76,15 +80,17 @@ ng_collect_rollouts \ ```bash # Terminal 1: Start servers source .venv/bin/activate -ng_run "+config_paths=[resources_servers/openenv/configs/openenv_maze.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env start \ + --resources-server openenv/openenv_maze \ + --model-type openai_model # Terminal 2: Collect rollouts source .venv/bin/activate -ng_collect_rollouts \ - +agent_name=openenv_maze_simple_agent \ - +input_jsonl_fpath=resources_servers/openenv/data/maze/example.jsonl \ - +output_jsonl_fpath=output_maze.jsonl \ - +num_samples_in_parallel=5 +gym eval run --no-serve \ + --agent openenv_maze_simple_agent \ + --input resources_servers/openenv/data/maze/example.jsonl \ + --output output_maze.jsonl \ + --concurrency 5 ``` ## Adding a New Environment @@ -178,15 +184,17 @@ Then start servers and collect rollouts: ```bash # Terminal 1: Start servers source .venv/bin/activate -ng_run "+config_paths=[resources_servers/openenv/configs/openenv_.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env start \ + --config resources_servers/openenv/configs/openenv_.yaml \ + --model-type openai_model # Terminal 2: Collect rollouts (after servers are ready) source .venv/bin/activate -ng_collect_rollouts \ - +agent_name=openenv__simple_agent \ - +input_jsonl_fpath=resources_servers/openenv/data//example.jsonl \ - +output_jsonl_fpath=output_.jsonl \ - +num_samples_in_parallel=5 +gym eval run --no-serve \ + --agent openenv__simple_agent \ + --input resources_servers/openenv/data//example.jsonl \ + --output output_.jsonl \ + --concurrency 5 ``` ## How It Works diff --git a/resources_servers/physics_judge/README.md b/resources_servers/physics_judge/README.md index 9444dbe3ed..05e0b9cf40 100644 --- a/resources_servers/physics_judge/README.md +++ b/resources_servers/physics_judge/README.md @@ -32,10 +32,10 @@ existing math_with_judge consumers (`aime24`, `aime25`, `gsm8k`, ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/physics_judge/configs/physics_judge.yaml,\ -resources_servers/physics_judge/configs/judge_openai.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server physics_judge \ + --resources-server physics_judge/judge_openai ``` The bundled `judge_openai.yaml` defaults the judge to `openai/gpt-oss-20b` @@ -53,11 +53,11 @@ CLI, or replace `judge_openai.yaml` with your own config that defines a ## Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=physics_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/physics_judge/data/example.jsonl \ - +output_jsonl_fpath=results/physics_judge_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent physics_judge_simple_agent \ + --input resources_servers/physics_judge/data/example.jsonl \ + --output results/physics_judge_rollouts.jsonl \ + --num-repeats 1 ``` ## Metrics diff --git a/resources_servers/polymath/README.md b/resources_servers/polymath/README.md index 8bae6bd138..b25c6d6633 100644 --- a/resources_servers/polymath/README.md +++ b/resources_servers/polymath/README.md @@ -30,19 +30,19 @@ PolyMath additions are at the metric-aggregation layer: ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/polymath/configs/polymath.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server polymath ``` ## Collecting rollouts (5-example smoke test) ```bash -ng_collect_rollouts \ - +agent_name=polymath_simple_agent \ - +input_jsonl_fpath=resources_servers/polymath/data/example.jsonl \ - +output_jsonl_fpath=results/polymath_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent polymath_simple_agent \ + --input resources_servers/polymath/data/example.jsonl \ + --output results/polymath_rollouts.jsonl \ + --num-repeats 1 ``` The server's vLLM endpoint should be started with diff --git a/resources_servers/rdkit_chemistry/README.md b/resources_servers/rdkit_chemistry/README.md index c34fe4b54a..cb10fe97d8 100644 --- a/resources_servers/rdkit_chemistry/README.md +++ b/resources_servers/rdkit_chemistry/README.md @@ -65,15 +65,14 @@ takes precedence over `use_box_format`. ## Example Usage ```bash -config_paths="resources_servers/rdkit_chemistry/configs/rdkit_chemistry.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[${config_paths}]" - -ng_collect_rollouts \ - +agent_name=rdkit_chemistry_agent \ - +input_jsonl_fpath=resources_servers/rdkit_chemistry/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/rdkit_chemistry/data/example_rollouts.jsonl +gym env start \ + --resources-server rdkit_chemistry \ + --model-type openai_model + +gym eval run --no-serve \ + --agent rdkit_chemistry_agent \ + --input resources_servers/rdkit_chemistry/data/example.jsonl \ + --output resources_servers/rdkit_chemistry/data/example_rollouts.jsonl ``` ## Licensing diff --git a/resources_servers/rdkit_chemistry/sandbox_launcher.py b/resources_servers/rdkit_chemistry/sandbox_launcher.py index 36bc14fedf..ba469196df 100644 --- a/resources_servers/rdkit_chemistry/sandbox_launcher.py +++ b/resources_servers/rdkit_chemistry/sandbox_launcher.py @@ -211,7 +211,7 @@ def start_sandbox( python = os.path.join(venv_path, "bin", "python") pip = os.path.join(venv_path, "bin", "pip") - # ng_run creates all server venvs in parallel. The ns_tools venv + # gym env start creates all server venvs in parallel. The ns_tools venv # may not be ready yet when rdkit_chemistry starts — wait for it. _wait_for_venv(python) @@ -298,13 +298,13 @@ def _run_startup_probe(port: int, timeout_s: float = _DEFAULT_STARTUP_PROBE_TIME logger.info("Sandbox startup probe passed on 127.0.0.1:%d", port) -_VENV_TIMEOUT = 600.0 # ng_run venv creation can take several minutes +_VENV_TIMEOUT = 600.0 # gym env start venv creation can take several minutes def _wait_for_venv(python: str) -> None: """Block until the venv's python binary exists and nemo_skills is importable. - ng_run creates all server venvs concurrently, so the ns_tools venv (which + gym env start creates all server venvs concurrently, so the ns_tools venv (which has nemo_skills) may still be installing when rdkit_chemistry starts. """ deadline = time.monotonic() + _VENV_TIMEOUT @@ -319,7 +319,7 @@ def _wait_for_venv(python: str) -> None: else: raise FileNotFoundError( f"Sandbox venv python not found at {python} after {_VENV_TIMEOUT}s. " - "Ensure ns_tools is part of the ng_run config." + "Ensure ns_tools is part of the gym env start config." ) phase = "nemo_skills" diff --git a/resources_servers/reasoning_gym/README.md b/resources_servers/reasoning_gym/README.md index 41ac8f2cc9..f206b49947 100644 --- a/resources_servers/reasoning_gym/README.md +++ b/resources_servers/reasoning_gym/README.md @@ -82,15 +82,17 @@ policy_model_name: nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 ## Launch nemo gym servers ```bash -ng_run "+config_paths=[resources_servers/reasoning_gym/configs/reasoning_gym.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --resources-server reasoning_gym \ + --model-type vllm_model ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=reasoning_gym_simple_agent \ - +input_jsonl_fpath=resources_servers/reasoning_gym/data/example.jsonl \ - +output_jsonl_fpath=results/reasoning_gym_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent reasoning_gym_simple_agent \ + --input resources_servers/reasoning_gym/data/example.jsonl \ + --output results/reasoning_gym_rollouts.jsonl \ + --limit 5 ``` \ No newline at end of file diff --git a/resources_servers/simpleqa/README.md b/resources_servers/simpleqa/README.md index 0267583de9..71a8492b31 100644 --- a/resources_servers/simpleqa/README.md +++ b/resources_servers/simpleqa/README.md @@ -39,20 +39,20 @@ the reasoning trace is split off before the response reaches this server. ## Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/simpleqa/configs/simpleqa.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --model-type vllm_model \ + --resources-server simpleqa \ +simpleqa.resources_servers.simpleqa.judge_model_server.name=policy_model ``` ## Collecting rollouts (5-example smoke test) ```bash -ng_collect_rollouts \ - +agent_name=simpleqa_simple_agent \ - +input_jsonl_fpath=resources_servers/simpleqa/data/example.jsonl \ - +output_jsonl_fpath=results/simpleqa_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent simpleqa_simple_agent \ + --input resources_servers/simpleqa/data/example.jsonl \ + --output results/simpleqa_rollouts.jsonl \ + --num-repeats 1 ``` ## Data Format diff --git a/resources_servers/single_step_tool_use_with_argument_comparison/README.md b/resources_servers/single_step_tool_use_with_argument_comparison/README.md index 7a489c15f4..ae730d4a14 100644 --- a/resources_servers/single_step_tool_use_with_argument_comparison/README.md +++ b/resources_servers/single_step_tool_use_with_argument_comparison/README.md @@ -8,17 +8,17 @@ Data links: ? ## Running servers The following command can be used to run this resources server, along with the tool simulation agent and an OpenAI model: ```bash -config_paths="resources_servers/single_step_tool_use_with_argument_comparison/configs/single_step_tool_use_with_argument_comparison.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --resources-server single_step_tool_use_with_argument_comparison \ + --model-type openai_model ``` Then, rollouts can be collected using a command such as the following: ```bash -ng_collect_rollouts \ - +agent_name=single_step_tool_use_with_argument_comparison_agent \ - +input_jsonl_fpath=resources_servers/single_step_tool_use_with_argument_comparison/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/single_step_tool_use_with_argument_comparison/data/example_rollouts.jsonl +gym eval run --no-serve \ + --agent single_step_tool_use_with_argument_comparison_agent \ + --input resources_servers/single_step_tool_use_with_argument_comparison/data/example.jsonl \ + --output resources_servers/single_step_tool_use_with_argument_comparison/data/example_rollouts.jsonl ``` # Licensing information diff --git a/resources_servers/speed_bench/README.md b/resources_servers/speed_bench/README.md index 132b6783cb..11a233986b 100644 --- a/resources_servers/speed_bench/README.md +++ b/resources_servers/speed_bench/README.md @@ -107,18 +107,18 @@ For an EAGLE3 / MTP setup with a paired draft model, see ```bash # Running servers — uses the demo local_vllm_model config above (drop in # your own model config to swap targets; just keep the speculative_config block). -config_paths="responses_api_models/local_vllm_model/configs/Qwen/Qwen3-30B-A3B-Instruct-2507-ngram-specdec.yaml,\ -resources_servers/speed_bench/configs/speed_bench.yaml,\ -responses_api_agents/speed_bench_agent/configs/speed_bench_agent.yaml" -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --model-type local_vllm_model/Qwen/Qwen3-30B-A3B-Instruct-2507-ngram-specdec \ + --resources-server speed_bench \ + --config responses_api_agents/speed_bench_agent/configs/speed_bench_agent.yaml \ +policy_model=Qwen3-30B-A3B-Instruct-2507-ngram-specdec # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=speed_bench_simple_agent \ - +input_jsonl_fpath=resources_servers/speed_bench/data/example.jsonl \ - +output_jsonl_fpath=results/speed_bench_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent speed_bench_simple_agent \ + --input resources_servers/speed_bench/data/example.jsonl \ + --output results/speed_bench_rollouts.jsonl \ + --num-repeats 1 ``` If you'd rather use the lighter-weight `vllm_model` config (which expects @@ -129,7 +129,7 @@ when starting the upstream `vllm serve` process. ## Tests ```bash -ng_test +entrypoint=resources_servers/speed_bench +gym env test --resources-server speed_bench ``` The unit tests cover the Prometheus parser, the running-delta math, the diff --git a/resources_servers/spider2_lite/README.md b/resources_servers/spider2_lite/README.md index 1d8795dd7e..6cd852814e 100644 --- a/resources_servers/spider2_lite/README.md +++ b/resources_servers/spider2_lite/README.md @@ -78,8 +78,10 @@ Each line must include either `gold_sql` (execute-and-compare) or `gold_result` Validate example data: ```bash -ng_prepare_data "+config_paths=[resources_servers/spider2_lite/configs/spider2_lite.yaml]" \ - +output_dirpath=/tmp/prepare +mode=example_validation +gym dataset collate \ + --resources-server spider2_lite \ + --output-dir /tmp/prepare \ + --mode example_validation ``` ## Known limitations diff --git a/resources_servers/structeval/README.md b/resources_servers/structeval/README.md index 6534ab7ab1..41c67db1e8 100644 --- a/resources_servers/structeval/README.md +++ b/resources_servers/structeval/README.md @@ -28,19 +28,19 @@ reward = 0.2 * render_score + 0.8 * key_validation_score ### Running servers ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/structeval/configs/structeval_nonrenderable.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --model-type vllm_model \ + --resources-server structeval/structeval_nonrenderable ``` ### Collecting rollouts ```bash -ng_collect_rollouts \ - +agent_name=structeval_nonrenderable_simple_agent \ - +input_jsonl_fpath=resources_servers/structeval/data/structeval_nonrenderable_train.jsonl \ - +output_jsonl_fpath=results/structeval_nonrenderable.jsonl \ - +resume_from_cache=True \ - +num_samples_in_parallel=256 +gym eval run --no-serve \ + --agent structeval_nonrenderable_simple_agent \ + --input resources_servers/structeval/data/structeval_nonrenderable_train.jsonl \ + --output results/structeval_nonrenderable.jsonl \ + --concurrency 256 \ + --resume ``` ## Data Preparation @@ -56,7 +56,7 @@ python resources_servers/structeval/misc/prepare_data.py \ ## Testing ```bash -ng_test +entrypoint=resources_servers/structeval +gym env test --resources-server structeval ``` ## Licensing diff --git a/resources_servers/structured_outputs/README.md b/resources_servers/structured_outputs/README.md index 50fa7d0c9e..3005dc8d03 100644 --- a/resources_servers/structured_outputs/README.md +++ b/resources_servers/structured_outputs/README.md @@ -55,37 +55,37 @@ text-output and tool-call variants is in The following command runs the text-output JSON config with the simple agent and an OpenAI model: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml, \ -resources_servers/structured_outputs/configs/structured_outputs_json.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --resources-server structured_outputs/structured_outputs_json ``` Then, text-output rollouts can be collected using a command such as: ```bash -ng_collect_rollouts \ - +agent_name=structured_outputs_simple_agent \ - +input_jsonl_fpath=resources_servers/structured_outputs/data/structured_outputs_260309_nano_v3_sdg_json_yaml_xml_val.jsonl \ - +output_jsonl_fpath=results/example_structured_outputs_json.jsonl \ - +resume_from_cache=True \ - +num_samples_in_parallel=256 +gym eval run --no-serve \ + --agent structured_outputs_simple_agent \ + --input resources_servers/structured_outputs/data/structured_outputs_260309_nano_v3_sdg_json_yaml_xml_val.jsonl \ + --output results/example_structured_outputs_json.jsonl \ + --concurrency 256 \ + --resume ``` For v4 tool-call structured outputs, use `structured_outputs_v4.yaml` and the v4 simple agent. The config routes through a non-executing agent because the emitted function call is the final answer, not an action to execute: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/structured_outputs/configs/structured_outputs_v4.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --model-type vllm_model \ + --resources-server structured_outputs/structured_outputs_v4 ``` ```bash -ng_collect_rollouts \ - +agent_name=structured_outputs_v4_simple_agent \ - +input_jsonl_fpath=resources_servers/structured_outputs/data/structured_outputs_v4_tool_call.jsonl \ - +output_jsonl_fpath=results/example_structured_outputs_v4_tool_call.jsonl \ - +resume_from_cache=True \ - +num_samples_in_parallel=256 +gym eval run --no-serve \ + --agent structured_outputs_v4_simple_agent \ + --input resources_servers/structured_outputs/data/structured_outputs_v4_tool_call.jsonl \ + --output results/example_structured_outputs_v4_tool_call.jsonl \ + --concurrency 256 \ + --resume ``` You can see breakdown of results from the rollout file using the provided breakdown_metrics file. @@ -98,32 +98,33 @@ python resources_servers/structured_outputs/misc/breakdown_rollouts_metrics.py \ ### Version 1 [251027] (JSON only) You can prepare the data for training with: ```bash -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/structured_outputs/configs/structured_outputs_json.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/structured_outputs \ - +mode=train_preparation +should_download=true +gym dataset collate \ + --config responses_api_models/openai_model/configs/openai_model.yaml \ + --resources-server structured_outputs/structured_outputs_json \ + --output-dir data/structured_outputs \ + --mode train_preparation \ + --download ``` ### Version 2 [260310] (JSON, YAML, XML) ```bash # prepare -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/structured_outputs/configs/structured_outputs_json_yaml_xml_v1.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/structured_outputs/ \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resources-server structured_outputs/structured_outputs_json_yaml_xml_v1 \ + --output-dir data/structured_outputs/ \ + --mode train_preparation \ + --download ``` ### Version 3 [260409] (JSON, YAML, XML, TOML, CSV) ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/structured_outputs/configs/structured_outputs_v3.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/structured_outputs_v3/ \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resources-server structured_outputs/structured_outputs_v3 \ + --output-dir data/structured_outputs_v3/ \ + --mode train_preparation \ + --download ``` ### Version 4 [260424] (Tool-call structured outputs) @@ -139,12 +140,12 @@ The uploaded GitLab dataset is: Prepare the v4 training data with: ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model_for_training.yaml,\ -resources_servers/structured_outputs/configs/structured_outputs_v4.yaml" -ng_prepare_data "+config_paths=[${config_paths}]" \ - +output_dirpath=data/structured_outputs_v4_tool_call/ \ - +mode=train_preparation \ - +should_download=true +gym dataset collate \ + --config responses_api_models/vllm_model/configs/vllm_model_for_training.yaml \ + --resources-server structured_outputs/structured_outputs_v4 \ + --output-dir data/structured_outputs_v4_tool_call/ \ + --mode train_preparation \ + --download ``` The v4 config wires the train dataset to @@ -166,7 +167,7 @@ Generation details and CLI examples are in # Testing ``` -ng_test +entrypoint=resources_servers/structured_outputs +gym env test --resources-server structured_outputs ``` # Licensing information diff --git a/resources_servers/swerl_gen/README.md b/resources_servers/swerl_gen/README.md index 2a672bd632..78f7b6a13b 100644 --- a/resources_servers/swerl_gen/README.md +++ b/resources_servers/swerl_gen/README.md @@ -60,19 +60,18 @@ You need to have singularity installed. Image having singularity: `/lustre/fsw/portfolios/llmservice/users/asohrabizade/codegen/sqsh/nvidian+nemo+verl_v2_enroot0.8.5.sqsh` ```bash -config_paths="responses_api_agents/simple_agent/configs/simple_agent.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/swerl_gen/configs/swerl_gen.yaml" +gym env start \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type openai_model \ + --resources-server swerl_gen -ng_run "+config_paths=[$config_paths]" - -ng_collect_rollouts \ - +agent_name=swerl_gen_simple_agent \ - +input_jsonl_fpath=resources_servers/swerl_gen/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/swerl_gen/data/example_rollouts.jsonl \ - +num_repeats=2 \ - +num_samples_in_parallel=4 \ - +responses_create_params.max_output_tokens=4096 +gym eval run --no-serve \ + --agent swerl_gen_simple_agent \ + --input resources_servers/swerl_gen/data/example.jsonl \ + --output resources_servers/swerl_gen/data/example_rollouts.jsonl \ + --num-repeats 2 \ + --concurrency 4 \ + --max-output-tokens 4096 ``` Rollout example diff --git a/resources_servers/swerl_llm_judge/README.md b/resources_servers/swerl_llm_judge/README.md index 33149a1a4a..da57ecea25 100644 --- a/resources_servers/swerl_llm_judge/README.md +++ b/resources_servers/swerl_llm_judge/README.md @@ -55,20 +55,18 @@ Notes: **Standard format (with `grading_mode`):** ```bash -config_paths="responses_api_agents/simple_agent/configs/simple_agent.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/swerl_llm_judge/configs/swerl_llm_judge.yaml" - - -ng_run "+config_paths=[$config_paths]" - -ng_collect_rollouts \ - +agent_name=swerl_llm_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/swerl_llm_judge/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/swerl_llm_judge/data/example_rollouts.jsonl \ - +num_repeats=1 \ - +num_samples_in_parallel=5 \ - +responses_create_params.max_output_tokens=4096 +gym env start \ + --config responses_api_agents/simple_agent/configs/simple_agent.yaml \ + --model-type openai_model \ + --resources-server swerl_llm_judge + +gym eval run --no-serve \ + --agent swerl_llm_judge_simple_agent \ + --input resources_servers/swerl_llm_judge/data/example.jsonl \ + --output resources_servers/swerl_llm_judge/data/example_rollouts.jsonl \ + --num-repeats 1 \ + --concurrency 5 \ + --max-output-tokens 4096 ``` diff --git a/resources_servers/tavily_search/README.md b/resources_servers/tavily_search/README.md index b4edfa0497..47fd6b3b05 100644 --- a/resources_servers/tavily_search/README.md +++ b/resources_servers/tavily_search/README.md @@ -27,29 +27,28 @@ tavily_search_resources_server: Commands to Run ``` -ng_download_dataset_from_gitlab \ - +dataset_name=tavily_search \ - +version=0.0.1 \ - +artifact_fpath=sft_samples_train.jsonl \ - +output_fpath=resources_servers/tavily_search/data/sft_samples/sft_samples_train.jsonl - -ng_download_dataset_from_gitlab \ - +dataset_name=tavily_search \ - +version=0.0.1 \ - +artifact_fpath=sft_samples_validation.jsonl \ - +output_fpath=resources_servers/tavily_search/data/sft_samples/sft_samples_validation.jsonl - -config_paths="resources_servers/tavily_search/configs/tavily_search_judge_vllm_model.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" - -ng_run "+config_paths=[${config_paths}]" +gym dataset download --storage gitlab \ + --name tavily_search \ + --revision 0.0.1 \ + --artifact sft_samples_train.jsonl \ + --output resources_servers/tavily_search/data/sft_samples/sft_samples_train.jsonl + +gym dataset download --storage gitlab \ + --name tavily_search \ + --revision 0.0.1 \ + --artifact sft_samples_validation.jsonl \ + --output resources_servers/tavily_search/data/sft_samples/sft_samples_validation.jsonl + +gym env start \ + --resources-server tavily_search/tavily_search_judge_vllm_model \ + --model-type vllm_model ``` ### Performance Metrics 100*16 samples: - Acc: 0.3212 -- Time in ng_collect: 44 mins +- Time in `gym eval run`: 44 mins # Licensing information diff --git a/resources_servers/terminus_judge/README.md b/resources_servers/terminus_judge/README.md index dd5dfbf3b5..808695adfe 100644 --- a/resources_servers/terminus_judge/README.md +++ b/resources_servers/terminus_judge/README.md @@ -32,19 +32,19 @@ For each verification request, the agent's JSON output is validated through mult The following command can be used to run this resources server, along with the simple agent and a policy model: ```bash -config_paths="resources_servers/terminus_judge/configs/terminus_judge.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[$config_paths]" \ +gym env start \ + --resources-server terminus_judge \ + --model-type openai_model \ +terminus_judge_resources_server.resources_servers.terminus_judge.judge_responses_create_params.max_output_tokens=512 ``` Then, rollouts can be collected using a command such as the following: ```bash -ng_collect_rollouts +agent_name=terminus_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/terminus_judge/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/terminus_judge/example_rollouts.jsonl +gym eval run --no-serve \ + --agent terminus_judge_simple_agent \ + --input resources_servers/terminus_judge/data/example.jsonl \ + --output resources_servers/terminus_judge/example_rollouts.jsonl ``` ## Expected Data Format diff --git a/resources_servers/terminus_judge/scripts/README.md b/resources_servers/terminus_judge/scripts/README.md index 99df28f8dd..c3547f0dbf 100644 --- a/resources_servers/terminus_judge/scripts/README.md +++ b/resources_servers/terminus_judge/scripts/README.md @@ -12,7 +12,7 @@ This pipeline uses the public `open-thoughts/OpenThoughts-Agent-v1-SFT` dataset - `datasets` - `openapi-schema-validator` 2. For smoke stage: - - `ng_run`, `ng_status`, `ng_collect_rollouts` on `PATH` + - `gym env start`, `gym env status`, `gym eval run --no-serve` on `PATH` - reachable policy/model endpoint 3. If HF cache is read-only: - `HF_HOME=/tmp/hf_home` diff --git a/resources_servers/text_to_sql/README.md b/resources_servers/text_to_sql/README.md index 741d423f82..6ca3c4af1e 100644 --- a/resources_servers/text_to_sql/README.md +++ b/resources_servers/text_to_sql/README.md @@ -68,19 +68,19 @@ Each data sample should include: ### Running Servers ```bash -config_paths="resources_servers/text_to_sql/configs/text_to_sql.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" - -ng_run "+config_paths=[${config_paths}]" \ +gym env start \ + --resources-server text_to_sql \ + --model-type openai_model \ +text_to_sql_resources_server.resources_servers.text_to_sql.judge_responses_create_params.max_output_tokens=512 ``` ### Collecting Rollouts ```bash -ng_collect_rollouts +agent_name=text_to_sql_simple_agent \ - +input_jsonl_fpath=resources_servers/text_to_sql/data/example.jsonl \ - +output_jsonl_fpath=resources_servers/text_to_sql/data/example_rollouts.jsonl +gym eval run --no-serve \ + --agent text_to_sql_simple_agent \ + --input resources_servers/text_to_sql/data/example.jsonl \ + --output resources_servers/text_to_sql/data/example_rollouts.jsonl ``` ## Configuration Options diff --git a/resources_servers/ugphysics_judge/README.md b/resources_servers/ugphysics_judge/README.md index 095d9bd2a2..7aca023e65 100644 --- a/resources_servers/ugphysics_judge/README.md +++ b/resources_servers/ugphysics_judge/README.md @@ -58,17 +58,17 @@ swap it for any other `responses_api_models/*` config that exposes a ```bash # Running servers -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/ugphysics_judge/configs/ugphysics_judge.yaml,\ -benchmarks/ugphysics/judge_gptoss20b.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server ugphysics_judge \ + --config benchmarks/ugphysics/judge_gptoss20b.yaml # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=ugphysics_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/ugphysics_judge/data/example.jsonl \ - +output_jsonl_fpath=results/ugphysics_judge_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent ugphysics_judge_simple_agent \ + --input resources_servers/ugphysics_judge/data/example.jsonl \ + --output results/ugphysics_judge_rollouts.jsonl \ + --num-repeats 1 ``` ## Metrics diff --git a/resources_servers/verifif/README.md b/resources_servers/verifif/README.md index 325451b6b5..918e7d8c0f 100644 --- a/resources_servers/verifif/README.md +++ b/resources_servers/verifif/README.md @@ -31,16 +31,18 @@ policy_model_name: gpt-5-2025-08-07 # or gpt-4.1-2025-04-14 ```bash cd /path/to/Gym source .venv/bin/activate -ng_run "+config_paths=[resources_servers/verifif/configs/verifif.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env start \ + --resources-server verifif \ + --model-type openai_model ``` ### 3. Run a test ```bash -ng_collect_rollouts \ - +agent_name=verifif_simple_agent \ - +input_jsonl_fpath=resources_servers/verifif/data/example.jsonl \ - +output_jsonl_fpath=results.jsonl +gym eval run --no-serve \ + --agent verifif_simple_agent \ + --input resources_servers/verifif/data/example.jsonl \ + --output results.jsonl ``` ## Architecture @@ -243,7 +245,7 @@ For high-throughput training: View server logs: ```bash -# Check terminal output from ng_run +# Check terminal output from gym env start # Or view Ray dashboard at http://127.0.0.1:8265 ``` diff --git a/resources_servers/vlm_eval_kit/README.md b/resources_servers/vlm_eval_kit/README.md index 32cdcccb0a..9b760edd23 100644 --- a/resources_servers/vlm_eval_kit/README.md +++ b/resources_servers/vlm_eval_kit/README.md @@ -54,9 +54,9 @@ python run.py --verbose \ ### Prepare data First run the VLMEvalKit server to install dependencies. ```bash -config_paths="resources_servers/vlm_eval_kit/configs/vlm_eval_kit.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --resources-server vlm_eval_kit \ + --model-type openai_model ``` Then cd into this directory and activate the Python environment @@ -75,15 +75,14 @@ python prepare_data.py ```bash WANDB_PROJECT= EXPERIMENT_NAME=vlmevalkit/gpt-4o-mini-20240718 -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/vlm_eval_kit/configs/vlm_eval_kit.yaml" -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ +gym eval run \ + --model-type openai_model \ + --resources-server vlm_eval_kit \ + --output results/$EXPERIMENT_NAME.jsonl \ + --split validation \ + --model gpt-4o-mini-2024-07-18 \ +wandb_project=$WANDB_PROJECT \ - +wandb_name=$EXPERIMENT_NAME \ - ++output_jsonl_fpath=results/$EXPERIMENT_NAME.jsonl \ - ++split=validation \ - ++policy_model_name=gpt-4o-mini-2024-07-18 + +wandb_name=$EXPERIMENT_NAME ``` # Licensing information diff --git a/resources_servers/wmt_translation/README.md b/resources_servers/wmt_translation/README.md index a649cbb04b..a99c31cc37 100644 --- a/resources_servers/wmt_translation/README.md +++ b/resources_servers/wmt_translation/README.md @@ -70,17 +70,17 @@ For an end-to-end SLURM run with COMET enabled, see the ```bash # Running servers (BLEU-only locally; flip compute_comet=true on cluster) -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/wmt_translation/configs/wmt_translation.yaml" -ng_run "+config_paths=[$config_paths]" \ - "++wmt_translation.resources_servers.wmt_translation.compute_comet=false" +gym env start \ + --model-type vllm_model \ + --resources-server wmt_translation \ + ++wmt_translation.resources_servers.wmt_translation.compute_comet=false # Collecting rollouts (5-example smoke test) -ng_collect_rollouts \ - +agent_name=wmt_translation_simple_agent \ - +input_jsonl_fpath=resources_servers/wmt_translation/data/example.jsonl \ - +output_jsonl_fpath=results/wmt_translation_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent wmt_translation_simple_agent \ + --input resources_servers/wmt_translation/data/example.jsonl \ + --output results/wmt_translation_rollouts.jsonl \ + --num-repeats 1 ``` For a fully reproducible end-to-end SLURM run that brings up vLLM with diff --git a/resources_servers/workplace_assistant/README.md b/resources_servers/workplace_assistant/README.md index c61e15945d..11c4afb8e4 100644 --- a/resources_servers/workplace_assistant/README.md +++ b/resources_servers/workplace_assistant/README.md @@ -11,17 +11,18 @@ Commands - Spin up server: ``` -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -resources_servers/workplace_assistant/configs/workplace_assistant.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type openai_model \ + --resources-server workplace_assistant ``` Collect trajectories: ``` -ng_collect_rollouts +agent_name=workplace_assistant_simple_agent \ - +input_jsonl_fpath=resources_servers/workplace_assistant/data/train.jsonl \ - +output_jsonl_fpath=results/workplace_assistant_trajectory_collection.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent workplace_assistant_simple_agent \ + --input resources_servers/workplace_assistant/data/train.jsonl \ + --output results/workplace_assistant_trajectory_collection.jsonl \ + --limit 1 ``` ## Generating Additional Training Data diff --git a/resources_servers/xlam_fc/README.md b/resources_servers/xlam_fc/README.md index 06978e9718..475abfd0bc 100644 --- a/resources_servers/xlam_fc/README.md +++ b/resources_servers/xlam_fc/README.md @@ -9,17 +9,17 @@ python resources_servers/xlam_fc/generate_dataset.py ``` ```bash -config_paths="responses_api_models/vllm_model/configs/vllm_model.yaml,\ -resources_servers/xlam_fc/configs/xlam_fc.yaml" -ng_run "+config_paths=[$config_paths]" +gym env start \ + --model-type vllm_model \ + --resources-server xlam_fc ``` ```bash -ng_collect_rollouts \ - +agent_name=xlam_fc_simple_agent \ - +input_jsonl_fpath=resources_servers/xlam_fc/data/train.jsonl \ - +output_jsonl_fpath=results/xlam_fc_trajectory_collection.jsonl \ - +limit=10 +gym eval run --no-serve \ + --agent xlam_fc_simple_agent \ + --input resources_servers/xlam_fc/data/train.jsonl \ + --output results/xlam_fc_trajectory_collection.jsonl \ + --limit 10 ``` ## Licensing diff --git a/resources_servers/xstest/README.md b/resources_servers/xstest/README.md index 11ef09bf23..80ef2af9a6 100644 --- a/resources_servers/xstest/README.md +++ b/resources_servers/xstest/README.md @@ -83,7 +83,7 @@ Edge cases: Recommended generation parameters for benchmarking: ```bash -ng_collect_rollouts ... "+responses_create_params={temperature: 1.0, top_p: 0.95, max_output_tokens: 32768}" +gym eval run --no-serve ... --temperature 1.0 --top-p 0.95 --max-output-tokens 32768 ``` Note: some models (e.g., Anthropic via Bedrock) do not allow `temperature` and `top_p` together. In that case, drop `top_p`. Use `temperature: 0.0` for deterministic/reproducible runs. @@ -100,16 +100,18 @@ contrast_privacy ### Example usage ```bash # For chat completions endpoints (vLLM, NIM, etc.): -ng_run "+config_paths=[resources_servers/xstest/configs/xstest.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --resources-server xstest \ + --model-type vllm_model # For OpenAI Responses API endpoints: -# ng_run "+config_paths=[resources_servers/xstest/configs/xstest.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +# gym env start --resources-server xstest --model-type openai_model -ng_collect_rollouts \ - +agent_name=xstest_simple_agent \ - +input_jsonl_fpath=resources_servers/xstest/data/example.jsonl \ - +output_jsonl_fpath=results/xstest_rollouts.jsonl \ - +num_repeats=1 +gym eval run --no-serve \ + --agent xstest_simple_agent \ + --input resources_servers/xstest/data/example.jsonl \ + --output results/xstest_rollouts.jsonl \ + --num-repeats 1 # Aggregate results python resources_servers/xstest/scripts/aggregate_results.py \ diff --git a/responses_api_agents/claude_code_agent/README.md b/responses_api_agents/claude_code_agent/README.md index 5ad1e4df19..94a0fabf3f 100644 --- a/responses_api_agents/claude_code_agent/README.md +++ b/responses_api_agents/claude_code_agent/README.md @@ -29,7 +29,7 @@ anthropic_base_url: http://localhost:8000 For a quick eval against Anthropic's API (or any endpoint set via `anthropic_base_url`), pass the resources server config, which includes the agent server config: ```bash -ng_run "+config_paths=[resources_servers/reasoning_gym/configs/reasoning_gym_claude_code_agent.yaml]" +gym env start --resources-server reasoning_gym/reasoning_gym_claude_code_agent ``` #### Against a Gym model server @@ -49,11 +49,11 @@ Use `vllm_model` for OpenAI-compatible **chat** endpoints (vLLM, NVIDIA build, m ### Run the agent ```bash -ng_collect_rollouts \ - +agent_name=reasoning_gym_claude_code_agent \ - +input_jsonl_fpath=resources_servers/reasoning_gym/data/example.jsonl \ - +output_jsonl_fpath=claude_code_rollout.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent reasoning_gym_claude_code_agent \ + --input resources_servers/reasoning_gym/data/example.jsonl \ + --output claude_code_rollout.jsonl \ + --limit 1 ``` For the model-server config above, use `+agent_name=reasoning_gym_claude_code_agent_model_server`. diff --git a/responses_api_agents/harbor_agent/README.md b/responses_api_agents/harbor_agent/README.md index 46e62cdcd0..223b10b325 100644 --- a/responses_api_agents/harbor_agent/README.md +++ b/responses_api_agents/harbor_agent/README.md @@ -200,9 +200,9 @@ export APPTAINER_DOCKER_PASSWORD= Then start NeMo Gym: ```bash -config_paths="responses_api_agents/harbor_agent/configs/harbor_agent.yaml,\ -responses_api_models/vllm_model/configs/vllm_model_for_training.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --config responses_api_agents/harbor_agent/configs/harbor_agent.yaml \ + --model-type vllm_model/vllm_model_for_training ``` ### 6) Test Harbor agent @@ -220,9 +220,10 @@ timestamped job directory containing per-trial outputs and a top-level ### 7) Collect rollouts ```bash -ng_collect_rollouts +agent_name=harbor_agent \ - +input_jsonl_fpath=responses_api_agents/harbor_agent/example/example_input.jsonl \ - +output_jsonl_fpath=responses_api_agents/harbor_agent/example/example_output.jsonl +gym eval run --no-serve \ + --agent harbor_agent \ + --input responses_api_agents/harbor_agent/example/example_input.jsonl \ + --output responses_api_agents/harbor_agent/example/example_output.jsonl ``` ### 8) View trajectories @@ -288,26 +289,29 @@ policy_model_name: Then follow the same Harbor-agent workflow with the Daytona config: ```bash -config_paths="responses_api_agents/harbor_agent/configs/harbor_agent_daytona.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" -ng_run "+config_paths=[${config_paths}]" +gym env start \ + --config responses_api_agents/harbor_agent/configs/harbor_agent_daytona.yaml \ + --model-type vllm_model ``` Alternatively, pass those values as CLI overrides: ```bash -ng_run "+config_paths=[${config_paths}]" \ - +policy_base_url= \ - +policy_api_key= \ - +policy_model_name= +gym env start \ + --config responses_api_agents/harbor_agent/configs/harbor_agent_daytona.yaml \ + --model-type vllm_model \ + --model-url \ + --model-api-key \ + --model ``` For five Terminal-Bench rollout inputs, use the checked-in input file: ```bash -ng_collect_rollouts +agent_name=harbor_agent \ - +input_jsonl_fpath=responses_api_agents/harbor_agent/example/terminal_bench_daytona_input.jsonl \ - +output_jsonl_fpath=/tmp/harbor_daytona_terminal_bench_output.jsonl +gym eval run --no-serve \ + --agent harbor_agent \ + --input responses_api_agents/harbor_agent/example/terminal_bench_daytona_input.jsonl \ + --output /tmp/harbor_daytona_terminal_bench_output.jsonl ``` Inspect the rollout and Harbor job directory: diff --git a/responses_api_agents/hermes_agent/README.md b/responses_api_agents/hermes_agent/README.md index 7fad332aca..951b3bfa72 100644 --- a/responses_api_agents/hermes_agent/README.md +++ b/responses_api_agents/hermes_agent/README.md @@ -13,17 +13,19 @@ policy_model_name: gpt-4o ## Launch nemo gym servers ```bash -ng_run "+config_paths=[resources_servers/math_with_judge/configs/math_with_judge_hermes_agent.yaml,responses_api_models/openai_model/configs/openai_model.yaml]" +gym env start \ + --resources-server math_with_judge/math_with_judge_hermes_agent \ + --model-type openai_model ``` ## Collect rollouts ```bash -ng_collect_rollouts \ - +agent_name=math_with_judge_hermes_agent \ - +input_jsonl_fpath=resources_servers/math_with_judge/data/example.jsonl \ - +output_jsonl_fpath=hermes_agent_rollout.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent math_with_judge_hermes_agent \ + --input resources_servers/math_with_judge/data/example.jsonl \ + --output hermes_agent_rollout.jsonl \ + --limit 1 ``` 5 example math rollouts are at `responses_api_agents/hermes_agent/data/` with statistics: diff --git a/responses_api_agents/langgraph_agent/README.md b/responses_api_agents/langgraph_agent/README.md index 99585f5786..6b256d9619 100644 --- a/responses_api_agents/langgraph_agent/README.md +++ b/responses_api_agents/langgraph_agent/README.md @@ -9,13 +9,15 @@ Please note that agents such as parallel thinking which produce non-monotonicall ## Quick Start ```bash -ng_run "+config_paths=[resources_servers/reasoning_gym/configs/reflection_agent.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --resources-server reasoning_gym/reflection_agent \ + --model-type vllm_model ``` ```bash -ng_collect_rollouts \ - +agent_name=reasoning_gym_reflection_agent \ - +input_jsonl_fpath=resources_servers/reasoning_gym/data/example.jsonl \ - +output_jsonl_fpath=example_rollouts.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent reasoning_gym_reflection_agent \ + --input resources_servers/reasoning_gym/data/example.jsonl \ + --output example_rollouts.jsonl \ + --limit 1 ``` diff --git a/responses_api_agents/mini_swe_agent/README.md b/responses_api_agents/mini_swe_agent/README.md index 370bfb29a5..ab5e12baaa 100644 --- a/responses_api_agents/mini_swe_agent/README.md +++ b/responses_api_agents/mini_swe_agent/README.md @@ -100,26 +100,28 @@ For how to download images and convert to .sif, you can refer to https://github. ```bash # Download swe-gym data -ng_download_dataset_from_gitlab \ - +dataset_name=mini_swe_agent \ - +version=0.0.1 \ - +artifact_fpath=train.jsonl \ - +output_fpath=data/train.jsonl +gym dataset download --storage gitlab \ + --name mini_swe_agent \ + --revision 0.0.1 \ + --artifact train.jsonl \ + --output data/train.jsonl # Start server -CONFIG_PATHS="resources_servers/mini_swe_agent/configs/mini_swe_agent.yaml,responses_api_models/openai_model/configs/openai_model.yaml" -ng_run +config_paths=[$CONFIG_PATHS] \ +gym env start \ + --resources-server mini_swe_agent \ + --model-type openai_model & \ '+mini_swe_simple_agent.responses_api_agents.mini_swe_agent.cache_dir_template=/path/to/images/xingyaoww_sweb.eval.x86_64.\{instance_id\}.sif' \ +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.run_golden=False \ +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.skip_if_exists=True \ +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.concurrency=16 \ +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.step_timeout=300 \ - +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.eval_timeout=900 & + +mini_swe_simple_agent.responses_api_agents.mini_swe_agent.eval_timeout=900 # Collect rollouts -ng_collect_rollouts +agent_name=mini_swe_simple_agent \ - +input_jsonl_fpath=data/train.jsonl \ - +output_jsonl_fpath=results/mini_swe_agent_swe_gym.jsonl +gym eval run --no-serve \ + --agent mini_swe_simple_agent \ + --input data/train.jsonl \ + --output results/mini_swe_agent_swe_gym.jsonl ``` ### Training Setup and Results diff --git a/responses_api_agents/stirrup_agent/README.md b/responses_api_agents/stirrup_agent/README.md index 4b53c0d40a..165dfa1f61 100644 --- a/responses_api_agents/stirrup_agent/README.md +++ b/responses_api_agents/stirrup_agent/README.md @@ -86,23 +86,22 @@ For each GDPVal task, the agent: The canonical entry point for GDPVal is the benchmark at [`benchmarks/gdpval/`](../../benchmarks/gdpval/README.md), which composes this agent with the GDPVal resources server and supports -`ng_prepare_benchmark` + `ng_e2e_collect_rollouts`: +`gym eval prepare` + `gym eval run`: ```bash # 1. Prepare the GDPVal benchmark JSONL. -ng_prepare_benchmark "+config_paths=[benchmarks/gdpval/config.yaml]" +gym eval prepare --benchmark gdpval # 2. Collect rollouts end-to-end (servers spin up automatically). -config_paths="responses_api_models/openai_model/configs/openai_model.yaml,\ -benchmarks/gdpval/config.yaml" JUDGE_API_KEY=... HF_TOKEN=... \ -ng_e2e_collect_rollouts \ - "+config_paths=[${config_paths}]" \ - ++split=benchmark \ - ++output_jsonl_fpath=results/gdpval_rubric.jsonl \ - ++policy_base_url=https://api.openai.com/v1 \ - ++policy_api_key=$OPENAI_API_KEY \ - ++policy_model_name=gpt-4.1-2025-04-14 +gym eval run \ + --model-type openai_model \ + --benchmark gdpval \ + --split benchmark \ + --output results/gdpval_rubric.jsonl \ + --model-url https://api.openai.com/v1 \ + --model-api-key $OPENAI_API_KEY \ + --model gpt-4.1-2025-04-14 ``` Each output line contains `responses_create_params`, the full `response`, a @@ -216,7 +215,7 @@ apptainer build gdpval.sif responses_api_agents/stirrup_agent/containers/gdpval. Then: ```yaml -# env.yaml or ng_run override +# env.yaml or gym env start override stirrup_agent: responses_api_agents: stirrup_agent: @@ -229,10 +228,11 @@ Pairwise comparison vs. a reference model is built into the GDPVal resources server (`resources_servers/gdpval`). Drive it from the benchmark config: ```bash -ng_e2e_collect_rollouts \ - "+config_paths=[responses_api_models/vllm_model/configs/vllm_model.yaml,benchmarks/gdpval/config.yaml]" \ - ++split=benchmark \ - ++output_jsonl_fpath=results/gdpval_compare.jsonl \ +gym eval run \ + --model-type vllm_model \ + --benchmark gdpval \ + --split benchmark \ + --output results/gdpval_compare.jsonl \ ++gdpval_resources_server.resources_servers.gdpval.reward_mode=comparison \ ++gdpval_resources_server.resources_servers.gdpval.reference_deliverables_dir=output/gdpval/reference-model ``` diff --git a/responses_api_agents/swe_agents/README.md b/responses_api_agents/swe_agents/README.md index 8a8b7efc7b..d7e4b4c668 100644 --- a/responses_api_agents/swe_agents/README.md +++ b/responses_api_agents/swe_agents/README.md @@ -251,15 +251,12 @@ policy_model_name: Qwen/Qwen3-Coder-30B-A3B-Instruct ### Step 2 — start the SWE-agents server ```bash -# OpenHands single-prompt -config_paths="responses_api_agents/swe_agents/configs/swebench_openhands.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" - -# Or full prompt × agent-class × tool-name diversity -config_paths="responses_api_agents/swe_agents/configs/swebench_multi_tools.yaml,\ -responses_api_models/vllm_model/configs/vllm_model.yaml" - -ng_run "+config_paths=[$config_paths]" \ +# OpenHands single-prompt (swap to +# responses_api_agents/swe_agents/configs/swebench_multi_tools.yaml for full +# prompt × agent-class × tool-name diversity) +gym env start \ + --config responses_api_agents/swe_agents/configs/swebench_openhands.yaml \ + --model-type vllm_model \ +swe_agents.responses_api_agents.swe_agents.container_formatter=/lustre/xxx/images/swe-bench/swebench_sweb.eval.x86_64.\{instance_id\}.sif \ +swe_agents.responses_api_agents.swe_agents.model_server.name=vllm_model ``` @@ -284,14 +281,16 @@ python responses_api_agents/swe_agents/client.py ## Batch evaluation / data collection ```bash -ng_collect_rollouts +agent_name=swe_agents \ - +input_jsonl_fpath=swebench-verified-converted.jsonl \ - +output_jsonl_fpath=swebench-verified.openhands.qwen3-30b-coder.jsonl \ - +model=Qwen/Qwen3-Coder-30B-A3B-Instruct \ - +temperature=0.7 +top_p=0.8 +gym eval run --no-serve \ + --agent swe_agents \ + --input swebench-verified-converted.jsonl \ + --output swebench-verified.openhands.qwen3-30b-coder.jsonl \ + --model Qwen/Qwen3-Coder-30B-A3B-Instruct \ + --temperature 0.7 \ + --top-p 0.8 ``` -`ng_collect_rollouts` defaults to a concurrency of 100; tune to your hardware. View the results with: +`gym eval run` defaults to a concurrency of 100; tune to your hardware. View the results with: ```bash ng_viewer +jsonl_fpath=swebench-verified.openhands.qwen3-30b-coder.jsonl diff --git a/responses_api_agents/tau2/README.md b/responses_api_agents/tau2/README.md index 5a3d2a3a26..8a8426e03f 100644 --- a/responses_api_agents/tau2/README.md +++ b/responses_api_agents/tau2/README.md @@ -1,8 +1,8 @@ # Description ```bash -config_paths="benchmarks/tau2/config.yaml,\ -responses_api_models/openai_model/configs/openai_model.yaml" -ng_run "+config_paths=[${config_paths}]" \ +gym env start \ + --benchmark tau2 \ + --model-type openai_model \ ++nemo_gym_log_dir=results/tau2 \ '++gpt-5_2-2025-12-11.responses_api_models.openai_model.openai_api_key=${openai_api_key}' \ '++gpt-5_2-2025-12-11.responses_api_models.openai_model.extra_body._delete_key=max_output_tokens' diff --git a/responses_api_agents/verifiers_agent/README.md b/responses_api_agents/verifiers_agent/README.md index 14a8f4a541..4502949311 100644 --- a/responses_api_agents/verifiers_agent/README.md +++ b/responses_api_agents/verifiers_agent/README.md @@ -21,14 +21,16 @@ policy_model_name: "Qwen/Qwen3-4B-Instruct-2507" ``` # start nemo gym servers -ng_run "+config_paths=[responses_api_agents/verifiers_agent/configs/acereason-math.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --config responses_api_agents/verifiers_agent/configs/acereason-math.yaml \ + --model-type vllm_model # generate a rollout -ng_collect_rollouts \ - +agent_name=verifiers_agent \ - +input_jsonl_fpath=responses_api_agents/verifiers_agent/data/acereason-math-example.jsonl \ - +output_jsonl_fpath=responses_api_agents/verifiers_agent/data/acereason-math-example-rollouts.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent verifiers_agent \ + --input responses_api_agents/verifiers_agent/data/acereason-math-example.jsonl \ + --output responses_api_agents/verifiers_agent/data/acereason-math-example-rollouts.jsonl \ + --limit 1 # view the rollout tail -n 1 responses_api_agents/verifiers_agent/data/acereason-math-example-rollouts.jsonl | jq | less @@ -98,21 +100,23 @@ deactivate source .venv/bin/activate # start nemo gym servers -ng_run "+config_paths=[responses_api_agents/verifiers_agent/configs/ascii-tree.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" +gym env start \ + --config responses_api_agents/verifiers_agent/configs/ascii-tree.yaml \ + --model-type vllm_model # generate a rollout -ng_collect_rollouts \ - +agent_name=verifiers_agent \ - +input_jsonl_fpath=responses_api_agents/verifiers_agent/data/ascii-tree-example.jsonl \ - +output_jsonl_fpath=responses_api_agents/verifiers_agent/data/ascii-tree-example-rollouts.jsonl \ - +limit=1 +gym eval run --no-serve \ + --agent verifiers_agent \ + --input responses_api_agents/verifiers_agent/data/ascii-tree-example.jsonl \ + --output responses_api_agents/verifiers_agent/data/ascii-tree-example-rollouts.jsonl \ + --limit 1 ``` ## Integration notes The patch to include prompt and generation token ids for preventing retokenization error when training with NeMo RL has been upstreamed into verifiers' `NeMoRLChatCompletionsClient`. We currently track verifiers `main` (`git+https://github.com/PrimeIntellect-ai/verifiers.git@main`) for this support; once verifiers `0.1.13` is released, the requirements pin will move to that tagged version. -For installing new prime environments and generating datasets, use a separate venv (outside of Gym) to avoid dependency conflicts with the `exclude-dependencies` section of Gym `pyproject.toml` and the server's pinned verifiers version. After generating your dataset, deactivate the separate venv and return to the Gym venv for running servers. Make sure to restart NeMo Gym servers with `ng_run` after any environment changes to ensure the pinned version of verifiers is used. +For installing new prime environments and generating datasets, use a separate venv (outside of Gym) to avoid dependency conflicts with the `exclude-dependencies` section of Gym `pyproject.toml` and the server's pinned verifiers version. After generating your dataset, deactivate the separate venv and return to the Gym venv for running servers. Make sure to restart NeMo Gym servers with `gym env start` after any environment changes to ensure the pinned version of verifiers is used. # Licensing information Code: Apache 2.0 diff --git a/responses_api_models/azure_openai_model/README.md b/responses_api_models/azure_openai_model/README.md index 4d795640cd..8be7b42a44 100644 --- a/responses_api_models/azure_openai_model/README.md +++ b/responses_api_models/azure_openai_model/README.md @@ -15,27 +15,26 @@ policy_model_name: gpt-5-nano ### Running the server Set the API version. It usually looks something like "2024-10-21". ```bash -config_paths="responses_api_models/azure_openai_model/configs/azure_openai_model.yaml, \ -resources_servers/equivalence_llm_judge/configs/equivalence_llm_judge.yaml" - -ng_run "+config_paths=[${config_paths}]" \ +gym env start \ + --model-type azure_openai_model \ + --resources-server equivalence_llm_judge \ +policy_model.responses_api_models.azure_openai_model.default_query.api-version= ``` ### Collecting Rollouts ```bash -ng_collect_rollouts \ - +agent_name=equivalence_llm_judge_simple_agent \ - +input_jsonl_fpath=resources_servers/equivalence_llm_judge/data/example.jsonl \ - +output_jsonl_fpath=results/example_rollouts.jsonl \ - +limit=5 +gym eval run --no-serve \ + --agent equivalence_llm_judge_simple_agent \ + --input resources_servers/equivalence_llm_judge/data/example.jsonl \ + --output results/example_rollouts.jsonl \ + --limit 5 ``` ### Test cases ```bash -ng_test +entrypoint=responses_api_models/azure_openai_model +gym env test +entrypoint=responses_api_models/azure_openai_model ``` ## Licensing information diff --git a/responses_api_models/local_vllm_model/README.md b/responses_api_models/local_vllm_model/README.md index 6fbbbe6989..d65ea81dd4 100644 --- a/responses_api_models/local_vllm_model/README.md +++ b/responses_api_models/local_vllm_model/README.md @@ -5,12 +5,12 @@ Run this on a single GPU node! Set tensor_parallel_size * data_parallel_size to the number of GPUs on your node. For this single node config, data_parallel_size_local is equal to data_parallel_size ```bash -config_paths="resources_servers/example_single_tool_call/configs/example_single_tool_call.yaml,\ -responses_api_models/local_vllm_model/configs/nano_v3_single_node.yaml" -ng_run "+config_paths=[${config_paths}]" \ +gym env start \ + --resources-server example_single_tool_call \ + --config responses_api_models/local_vllm_model/configs/nano_v3_single_node.yaml &> temp.log & \ ++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.tensor_parallel_size=4 \ ++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.data_parallel_size=2 \ - ++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.data_parallel_size_local=2 &> temp.log & + ++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.data_parallel_size_local=2 ``` View the logs diff --git a/responses_api_models/local_vllm_model/configs/local_vllm_model.yaml b/responses_api_models/local_vllm_model/configs/local_vllm_model.yaml new file mode 100644 index 0000000000..09ff537d96 --- /dev/null +++ b/responses_api_models/local_vllm_model/configs/local_vllm_model.yaml @@ -0,0 +1,26 @@ +# Generic local vLLM deployment. Serves the checkpoint passed via `--model ` +# (i.e. `policy_model_name`). Set parallelism to match your node before running. +# +# Example: +# gym eval run --benchmark aime24 --model-type local_vllm_model --model Qwen/Qwen3-30B-A3B-Instruct-2507 +policy_model: + responses_api_models: + local_vllm_model: + entrypoint: app.py + model: ${policy_model_name} + return_token_id_information: false + uses_reasoning_parser: false + + # If your model is downloaded at ~/.cache/huggingface/hub/models----, set hf_home to ~/.cache/huggingface. + hf_home: null + + vllm_serve_env_vars: {} + + # vLLM parallelism is sensitive; tensor_parallel_size * pipeline_parallel_size GPUs are used per model instance. + # Override per node, e.g. `++policy_model.responses_api_models.local_vllm_model.vllm_serve_kwargs.tensor_parallel_size=8`. + vllm_serve_kwargs: + data_parallel_size: 1 + tensor_parallel_size: 1 + pipeline_parallel_size: 1 + trust_remote_code: true + gpu_memory_utilization: 0.9 diff --git a/responses_api_models/local_vllm_model_proxy/README.md b/responses_api_models/local_vllm_model_proxy/README.md index d4e57d6dd0..df835d45b5 100644 --- a/responses_api_models/local_vllm_model_proxy/README.md +++ b/responses_api_models/local_vllm_model_proxy/README.md @@ -3,9 +3,9 @@ # End-to-end test with GPT OSS 20B reasoning high ```bash -config_paths="responses_api_models/local_vllm_model/configs/openai/gpt-oss-20b-reasoning-high.yaml,\ -responses_api_models/local_vllm_model_proxy/configs/local_vllm_model_proxy.yaml" -ng_run "+config_paths=[${config_paths}]" \ +gym env start \ + --model-type local_vllm_model/openai/gpt-oss-20b-reasoning-high \ + --model-type local_vllm_model_proxy \ ++policy_model_proxy.responses_api_models.local_vllm_model_proxy.model_server.name=gpt-oss-20b-reasoning-high \ ++policy_model_proxy.responses_api_models.local_vllm_model_proxy.extra_body.max_tokens=10 ``` diff --git a/tests/unit_tests/test_benchmarks.py b/tests/unit_tests/test_benchmarks.py index 02e8ecb9cb..e0c5ece28f 100644 --- a/tests/unit_tests/test_benchmarks.py +++ b/tests/unit_tests/test_benchmarks.py @@ -19,7 +19,7 @@ from omegaconf import OmegaConf from yaml import safe_load -from nemo_gym.benchmarks import list_benchmarks, prepare_benchmark +from nemo_gym.cli.eval import _benchmark_extras, _fuzzy_matches, list_benchmarks, prepare_benchmark def _mock_global_config(config: dict = None): @@ -29,18 +29,119 @@ def _mock_global_config(config: dict = None): class TestListBenchmarks: def test_lists_found_benchmarks(self, capsys) -> None: - with patch("nemo_gym.benchmarks.get_global_config_dict", return_value=_mock_global_config()): + with patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config()): list_benchmarks() assert "aime24" in capsys.readouterr().out def test_no_benchmarks(self, capsys) -> None: with ( - patch("nemo_gym.benchmarks.get_global_config_dict", return_value=_mock_global_config()), - patch("nemo_gym.benchmarks._load_benchmarks_from_config_paths", return_value={}), + patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config()), + patch("nemo_gym.cli.eval._load_benchmarks_from_config_paths", return_value={}), ): list_benchmarks() assert "No benchmarks found" in capsys.readouterr().out + def test_json_output(self, capsys) -> None: + import json + + bench = MagicMock(agent_name="my_agent", num_repeats=4) + with ( + patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config({"json": True})), + patch("nemo_gym.cli.eval._load_benchmarks_from_config_paths", return_value={"my_bench": bench}), + patch("nemo_gym.cli.eval._benchmark_extras", return_value=("math", ["math_with_judge"])), + ): + list_benchmarks() + assert json.loads(capsys.readouterr().out) == [ + {"name": "my_bench", "agent_name": "my_agent", "domain": "math", "num_repeats": 4} + ] + + def test_json_output_empty(self, capsys) -> None: + import json + + with ( + patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config({"json": True})), + patch("nemo_gym.cli.eval._load_benchmarks_from_config_paths", return_value={}), + ): + list_benchmarks() + assert json.loads(capsys.readouterr().out) == [] + + +class TestFuzzyMatches: + def test_substring_matches(self) -> None: + assert _fuzzy_matches("math", "math_with_judge") + + def test_token_typo_matches(self) -> None: + # `aimee` is a near-miss for the `aime` token in `aime24`. + assert _fuzzy_matches("aimee", "aime24") + + def test_matches_against_agent_field(self) -> None: + assert _fuzzy_matches("judge", "aime24", "math_with_judge_agent") + + def test_skips_empty_fields(self) -> None: + assert not _fuzzy_matches("math", "", None) + + def test_no_match(self) -> None: + assert not _fuzzy_matches("zzznomatch", "aime24", "math_with_judge") + + +class TestBenchmarkExtras: + def test_resolves_domain_and_terms_from_real_config(self) -> None: + from nemo_gym.benchmarks import BENCHMARKS_DIR, BenchmarkConfig + + bench = BenchmarkConfig.from_config_path(BENCHMARKS_DIR / "aime24" / "config.yaml") + domain, terms = _benchmark_extras(bench) + + assert domain == "math" + # The resources server's inner name and the dataset name are recovered for search. + assert "math_with_judge" in terms + assert "aime24" in terms + + +class TestSearchBenchmarks: + # Map each benchmark name to the (domain, extra terms) its config would resolve to. + # As in the real resolver, `domain` is also included among the search terms. + EXTRAS = { + "aime24": ("math", ["aime24_math_resources_server", "math_with_judge", "aime24", "math"]), + "gpqa_diamond": ("science", ["gpqa_resources_server", "gpqa", "gpqa_diamond", "science"]), + } + + def _bench(self, key: str): + bench = MagicMock(agent_name="my_agent", num_repeats=1) + bench.config_key = key # let the patched _benchmark_extras find the right entry + return bench + + def _benchmarks(self) -> dict: + return {name: self._bench(name) for name in self.EXTRAS} + + def _run(self, query: str, benchmarks: dict, capsys) -> str: + with ( + patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config({"query": query})), + patch("nemo_gym.cli.eval._load_benchmarks_from_config_paths", return_value=benchmarks), + patch("nemo_gym.cli.eval._benchmark_extras", side_effect=lambda b: self.EXTRAS[b.config_key]), + ): + list_benchmarks() + return capsys.readouterr().out + + def test_query_filters_by_name(self, capsys) -> None: + out = self._run("aime", self._benchmarks(), capsys) + assert "aime24" in out + assert "gpqa" not in out + + def test_query_matches_domain(self, capsys) -> None: + # "science" only appears via gpqa's domain, not its name/agent. + out = self._run("science", self._benchmarks(), capsys) + assert "gpqa_diamond" in out + assert "aime24" not in out + + def test_query_matches_resource_server(self, capsys) -> None: + # "judge" only appears via aime24's resources server name. + out = self._run("judge", self._benchmarks(), capsys) + assert "aime24" in out + assert "gpqa_diamond" not in out + + def test_query_no_match_message(self, capsys) -> None: + assert "No benchmarks match 'zzz'" in self._run("zzz", self._benchmarks(), capsys) + class TestPrepareBenchmark: def _make_bench_dir(self, tmp_path: Path, name: str = "fake_bench") -> tuple[Path, Path]: @@ -73,13 +174,13 @@ def test_calls_prepare(self, tmp_path: Path) -> None: with ( patch( - "nemo_gym.benchmarks.get_global_config_dict", + "nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config( {"config_paths": [str(config_path)], **safe_load(config_path.read_text())} ), ), - patch("nemo_gym.benchmarks.BENCHMARKS_DIR", bench_dir.parent), - patch("nemo_gym.benchmarks.importlib.import_module", return_value=mock_module), + patch("nemo_gym.cli.eval.BENCHMARKS_DIR", bench_dir.parent), + patch("nemo_gym.cli.eval.importlib.import_module", return_value=mock_module), ): prepare_benchmark() mock_module.prepare.assert_called_once() @@ -90,12 +191,12 @@ def test_missing_prepare_py(self, tmp_path: Path) -> None: with ( patch( - "nemo_gym.benchmarks.get_global_config_dict", + "nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config( {"config_paths": [str(config_path)], **safe_load(config_path.read_text())} ), ), - patch("nemo_gym.benchmarks.BENCHMARKS_DIR", bench_dir.parent), + patch("nemo_gym.cli.eval.BENCHMARKS_DIR", bench_dir.parent), ): with pytest.raises(RuntimeError, match="The following benchmarks are missing a valid prepare script"): prepare_benchmark() @@ -107,13 +208,13 @@ def test_missing_prepare_function(self, tmp_path: Path) -> None: with ( patch( - "nemo_gym.benchmarks.get_global_config_dict", + "nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config( {"config_paths": [str(config_path)], **safe_load(config_path.read_text())} ), ), - patch("nemo_gym.benchmarks.BENCHMARKS_DIR", bench_dir.parent), - patch("nemo_gym.benchmarks.importlib.import_module", return_value=mock_module), + patch("nemo_gym.cli.eval.BENCHMARKS_DIR", bench_dir.parent), + patch("nemo_gym.cli.eval.importlib.import_module", return_value=mock_module), ): with pytest.raises( AssertionError, @@ -124,12 +225,29 @@ def test_missing_prepare_function(self, tmp_path: Path) -> None: def test_no_benchmark_in_config_paths(self) -> None: with ( patch( - "nemo_gym.benchmarks.get_global_config_dict", + "nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config({"config_paths": ["resources_servers/foo/configs/foo.yaml"]}), ), - patch("nemo_gym.benchmarks._load_benchmarks_from_config_paths", return_value={}), + patch("nemo_gym.cli.eval._load_benchmarks_from_config_paths", return_value={}), ): - with pytest.raises(AssertionError, match="No benchmark config found in config_paths"): + with pytest.raises(AssertionError, match="No benchmark config found"): + prepare_benchmark() + + def test_no_benchmark_dataset_reports_inspected_instances(self, tmp_path: Path) -> None: + # A server instance is present but declares no `benchmark` dataset; the error should name it + # so the user can see what was inspected. + config = { + "config_paths": ["benchmarks/dummy/config.yaml"], + "dummy_agent": { + "responses_api_agents": { + "simple_agent": { + "datasets": [{"name": "not_a_benchmark", "type": "train", "jsonl_fpath": str(tmp_path)}] + } + } + }, + } + with patch("nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config(config)): + with pytest.raises(AssertionError, match=r"Inspected server instances \['dummy_agent'\]"): prepare_benchmark() def test_caching_sanity(self, tmp_path: Path) -> None: @@ -141,7 +259,7 @@ def test_caching_sanity(self, tmp_path: Path) -> None: with ( patch( - "nemo_gym.benchmarks.get_global_config_dict", + "nemo_gym.cli.eval.get_global_config_dict", return_value=_mock_global_config( { "use_cached_prepared_benchmarks": True, @@ -150,8 +268,8 @@ def test_caching_sanity(self, tmp_path: Path) -> None: } ), ), - patch("nemo_gym.benchmarks.BENCHMARKS_DIR", bench_dir.parent), - patch("nemo_gym.benchmarks.importlib.import_module", return_value=mock_module), + patch("nemo_gym.cli.eval.BENCHMARKS_DIR", bench_dir.parent), + patch("nemo_gym.cli.eval.importlib.import_module", return_value=mock_module), ): prepare_benchmark() diff --git a/tests/unit_tests/test_cli.py b/tests/unit_tests/test_cli.py index 3236185836..8e3d7edd7d 100644 --- a/tests/unit_tests/test_cli.py +++ b/tests/unit_tests/test_cli.py @@ -13,10 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import shutil -import sys import tomllib from importlib import import_module -from io import StringIO from pathlib import Path from subprocess import TimeoutExpired from unittest.mock import MagicMock, patch @@ -26,13 +24,12 @@ import nemo_gym.global_config from nemo_gym import PARENT_DIR -from nemo_gym.cli import ( +from nemo_gym.cli.env import ( _FORCE_KILL_REAP_TIMEOUT_SEC, _GRACEFUL_SHUTDOWN_TIMEOUT_SEC, RunConfig, RunHelper, _select_shard, - display_help, init_resources_server, ) from nemo_gym.config_types import ResourcesServerInstanceConfig @@ -75,46 +72,16 @@ class TestCLI: def test_sanity(self) -> None: RunConfig(entrypoint="", name="") - def test_pyproject_scripts(self) -> None: + def test_pyproject_scripts_are_importable(self) -> None: + """Every console-script entry point must resolve to an importable callable.""" pyproject_path = PARENT_DIR / "pyproject.toml" with pyproject_path.open("rb") as f: pyproject_data = tomllib.load(f) - project_scripts = pyproject_data["project"]["scripts"] - - for script_name, import_path in project_scripts.items(): - # Dedupe `nemo_gym_*` from `ng_*` commands - if not script_name.startswith("ng_"): - continue - - # We only test `+h=true` and not `+help=true` - print(f"Running `{script_name} +h=true`") - + for script_name, import_path in pyproject_data["project"]["scripts"].items(): module, fn = import_path.split(":") - fn = getattr(import_module(module), fn) - - with MonkeyPatch.context() as mp: - mp.setattr(nemo_gym.global_config, "_GLOBAL_CONFIG_DICT", OmegaConf.create({"h": True})) - - text_trap = StringIO() - mp.setattr(sys, "stdout", text_trap) - - with raises(SystemExit): - fn() - - def test_display_help_discovers_scripts(self) -> None: - with MonkeyPatch.context() as mp: - mp.setattr(nemo_gym.global_config, "_GLOBAL_CONFIG_DICT", OmegaConf.create({})) - - text_trap = StringIO() - mp.setattr(sys, "stdout", text_trap) - - display_help() - - output = text_trap.getvalue() - assert "ng_help" in output - assert "ng_run" in output - assert "ng_collect_rollouts" in output + target = getattr(import_module(module), fn) + assert callable(target), f"{script_name} -> {import_path} is not callable" def test_init_resources_server_includes_domain(self) -> None: """Test that init_resources_server creates a config with the required domain field.""" diff --git a/tests/unit_tests/test_cli_compat.py b/tests/unit_tests/test_cli_compat.py new file mode 100644 index 0000000000..758d662c5b --- /dev/null +++ b/tests/unit_tests/test_cli_compat.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The CLI refactor moved these symbols into `nemo_gym.cli`; the old import paths must keep +working (with a `DeprecationWarning`) for the deprecation period. Remove once they are dropped.""" + +import importlib + +import pytest + + +# (old_module, old_name, new_module, new_name) for each relocated symbol. +MOVED_SYMBOLS = [ + ("nemo_gym.benchmarks", "list_benchmarks", "nemo_gym.cli.eval", "list_benchmarks"), + ("nemo_gym.benchmarks", "PrepareBenchmarkConfig", "nemo_gym.cli.eval", "PrepareBenchmarkConfig"), + ("nemo_gym.benchmarks", "prepare_benchmark", "nemo_gym.cli.eval", "prepare_benchmark"), + ( + "nemo_gym.dataset_orchestrator", + "upload_jsonl_dataset_to_hf_cli", + "nemo_gym.cli.dataset", + "upload_jsonl_dataset_to_hf_cli", + ), + ( + "nemo_gym.dataset_orchestrator", + "upload_jsonl_dataset_to_hf_and_delete_gitlab_cli", + "nemo_gym.cli.dataset", + "upload_jsonl_dataset_to_hf_and_delete_gitlab_cli", + ), + ( + "nemo_gym.dataset_orchestrator", + "download_jsonl_dataset_from_hf_cli", + "nemo_gym.cli.dataset", + "download_jsonl_dataset_from_hf_cli", + ), + ( + "nemo_gym.dataset_orchestrator", + "delete_jsonl_dataset_from_gitlab_cli", + "nemo_gym.cli.dataset", + "delete_jsonl_dataset_from_gitlab_cli", + ), + ("nemo_gym.gitlab_utils", "upload_jsonl_dataset_cli", "nemo_gym.cli.dataset", "upload_jsonl_dataset_cli"), + ("nemo_gym.gitlab_utils", "download_jsonl_dataset_cli", "nemo_gym.cli.dataset", "download_jsonl_dataset_cli"), + ("nemo_gym.prompt", "materialize_prompts_cli", "nemo_gym.cli.dataset", "materialize_prompts_cli"), + ("nemo_gym.reward_profile", "reward_profile", "nemo_gym.cli.eval", "reward_profile"), + ("nemo_gym.rollout_collection", "collect_rollouts", "nemo_gym.cli.eval", "collect_rollouts"), + ("nemo_gym.rollout_collection", "aggregate_rollouts", "nemo_gym.cli.eval", "aggregate_rollouts"), + ("nemo_gym.train_data_utils", "prepare_data", "nemo_gym.cli.dataset", "prepare_data"), + ("nemo_gym.cli", "RunHelper", "nemo_gym.cli.env", "RunHelper"), + ("nemo_gym.cli", "RunConfig", "nemo_gym.cli.env", "RunConfig"), + ("nemo_gym.cli", "TestConfig", "nemo_gym.cli.env", "TestConfig"), + ("nemo_gym.cli", "TestAllConfig", "nemo_gym.cli.env", "TestAllConfig"), + ("nemo_gym.cli", "PipListConfig", "nemo_gym.cli.env", "PipListConfig"), + ("nemo_gym.cli", "run", "nemo_gym.cli.env", "run"), + ("nemo_gym.cli", "test", "nemo_gym.cli.env", "test"), + ("nemo_gym.cli", "test_all", "nemo_gym.cli.env", "test_all"), + ("nemo_gym.cli", "init_resources_server", "nemo_gym.cli.env", "init_resources_server"), + ("nemo_gym.cli", "dump_config", "nemo_gym.cli.env", "dump_config"), + ("nemo_gym.cli", "status", "nemo_gym.cli.env", "status"), + ("nemo_gym.cli", "pip_list", "nemo_gym.cli.env", "pip_list"), + ("nemo_gym.cli", "e2e_rollout_collection", "nemo_gym.cli.eval", "e2e_rollout_collection"), + ("nemo_gym.cli", "dev_test", "nemo_gym.cli.dev", "dev_test"), + ("nemo_gym.cli", "VersionConfig", "nemo_gym.cli.general", "VersionConfig"), + ("nemo_gym.cli", "version", "nemo_gym.cli.general", "version"), + ("nemo_gym.cli", "GlobalConfigDictParserConfig", "nemo_gym.global_config", "GlobalConfigDictParserConfig"), + ("nemo_gym.cli_setup_command", "run_command", "nemo_gym.cli.setup_command", "run_command"), + ("nemo_gym.cli_setup_command", "setup_env_command", "nemo_gym.cli.setup_command", "setup_env_command"), +] + + +@pytest.mark.parametrize("old_module, old_name, new_module, new_name", MOVED_SYMBOLS) +def test_deprecated_import_still_resolves_and_warns(old_module, old_name, new_module, new_name) -> None: + old_mod = importlib.import_module(old_module) + with pytest.warns(DeprecationWarning): + resolved = getattr(old_mod, old_name) + assert resolved is getattr(importlib.import_module(new_module), new_name) diff --git a/tests/unit_tests/test_cli_legacy.py b/tests/unit_tests/test_cli_legacy.py new file mode 100644 index 0000000000..4fe01e6f9d --- /dev/null +++ b/tests/unit_tests/test_cli_legacy.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import tomllib + +import pytest +from pytest import MonkeyPatch + +import nemo_gym.cli.legacy as legacy +import nemo_gym.cli.main as cli_main +from nemo_gym import PARENT_DIR + + +def _legacy_scripts() -> list[tuple[str, str]]: + """All console scripts whose name is a legacy ng_*/nemo_gym_* alias.""" + with (PARENT_DIR / "pyproject.toml").open("rb") as f: + scripts = tomllib.load(f)["project"]["scripts"] + return [(name, target) for name, target in scripts.items() if name.startswith(("ng_", "nemo_gym_"))] + + +LEGACY_SCRIPTS = _legacy_scripts() + + +class TestLegacyDeprecation: + """Remove these tests once the legacy commands are removed.""" + + def test_legacy_scripts_were_discovered(self) -> None: + # Guard so the parametrized test below can't pass vacuously if discovery breaks. + assert len(LEGACY_SCRIPTS) > 1 + + @pytest.mark.parametrize("name, target", LEGACY_SCRIPTS) + def test_legacy_command_shows_deprecation(self, monkeypatch: MonkeyPatch, capsys, name: str, target: str) -> None: + # Every legacy alias must route through the shim, which prints a deprecation notice and keeps working. + assert target == "nemo_gym.cli.legacy:main", f"{name} should route through the legacy shim" + + # Stub the actual execution paths so nothing real runs. + monkeypatch.setattr(legacy, "gym_main", lambda: None) + monkeypatch.setattr(legacy, "dispatch", lambda *a, **k: None) + monkeypatch.setattr(sys, "argv", [name]) + + legacy.main() + + assert "deprecated" in capsys.readouterr().err + + def test_legacy_mapping_is_populated(self) -> None: + # Guard so the parametrized test below can't pass vacuously if LEGACY is emptied. + assert len(legacy.LEGACY) > 1 + + @pytest.mark.parametrize("key, tokens", list(legacy.LEGACY.items())) + def test_legacy_tokens_resolve_to_real_gym_command( + self, monkeypatch: MonkeyPatch, key: str, tokens: list[str] + ) -> None: + # The tests above confirm each alias routes through the shim, but not that the mapped tokens + # are a real `gym` command. Feed every mapping through the actual parser so a typo in the + # mapping (e.g. ["env", "ruun"]) fails here instead of only at user runtime. + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", *tokens]) + try: + cli_main.main() + except SystemExit as exc: + # `gym --help` is the only mapping that legitimately exits (argparse prints help, exit 0). + assert tokens == ["--help"], f"`gym {' '.join(tokens)}` (legacy `{key}`) is not a valid command" + assert exc.code == 0 + + def test_unknown_alias_exits_nonzero(self, monkeypatch: MonkeyPatch, capsys) -> None: + # An alias with no LEGACY mapping (stale script or user typo) must fail loudly, not KeyError. + monkeypatch.setattr(legacy, "gym_main", lambda: None) + monkeypatch.setattr(legacy, "dispatch", lambda *a, **k: None) + monkeypatch.setattr(sys, "argv", ["ng_does_not_exist"]) + + with pytest.raises(SystemExit) as exc_info: + legacy.main() + + assert exc_info.value.code == 1 + assert "no known `gym` equivalent" in capsys.readouterr().err diff --git a/tests/unit_tests/test_cli_main.py b/tests/unit_tests/test_cli_main.py new file mode 100644 index 0000000000..de9bed828f --- /dev/null +++ b/tests/unit_tests/test_cli_main.py @@ -0,0 +1,935 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import sys +import types + +import pytest +from pytest import MonkeyPatch + +import nemo_gym.cli.main as cli_main +import nemo_gym.global_config as gc +from nemo_gym import WORKING_DIR +from nemo_gym.cli.main import main +from nemo_gym.global_config import NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME + + +def _dispatch_for(monkeypatch: MonkeyPatch, argv: list[str]) -> tuple[str, list[str]]: + """Run the gym router for `argv` and return the (target, overrides) handed to dispatch.""" + captured: dict = {} + + def fake_dispatch(target: str, overrides: list[str]) -> None: + captured["target"] = target + captured["overrides"] = overrides + + monkeypatch.setattr(cli_main, "dispatch", fake_dispatch) + monkeypatch.setattr(sys, "argv", ["gym", *argv]) + main() + return captured["target"], captured["overrides"] + + +def _split_overrides(overrides: list[str]) -> tuple[set[str], set[str]]: + """Split overrides into (config paths, other overrides) as sets, so tests never assert ordering.""" + prefix = "+config_paths=[" + config_tokens = [o for o in overrides if o.startswith(prefix) and o.endswith("]")] + assert len(config_tokens) <= 1 # --config and the asset selectors coalesce into a single token + paths = set(config_tokens[0][len(prefix) : -1].split(",")) if config_tokens else set() + others = {o for o in overrides if o not in config_tokens} + return paths, others + + +# `gym ` -> the legacy ng_ function it dispatches to, for the config-accepting commands. +CONFIG_COMMANDS = [ + (["env", "start"], "nemo_gym.cli.env:run"), + (["env", "resolve"], "nemo_gym.cli.env:dump_config"), + (["eval", "prepare"], "nemo_gym.cli.eval:prepare_benchmark"), + (["eval", "aggregate"], "nemo_gym.cli.eval:aggregate_rollouts"), + (["eval", "run"], "nemo_gym.cli.eval:e2e_rollout_collection"), + (["dataset", "collate"], "nemo_gym.cli.dataset:prepare_data"), +] + + +class TestConfigFlag: + @pytest.mark.parametrize("command, expected_target", CONFIG_COMMANDS) + def test_config_becomes_config_paths(self, monkeypatch: MonkeyPatch, command, expected_target) -> None: + """`gym --config X` dispatches to ng_ with +config_paths=[X].""" + target, overrides = _dispatch_for(monkeypatch, [*command, "--config", "my.yaml"]) + assert target == expected_target + assert overrides == ["+config_paths=[my.yaml]"] + + def test_repeated_config_joined_into_one_list(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "start", "--config", "a.yaml", "--config", "b.yaml"]) + + # We have this set of asserts to avoid asserting configs order in the string + assert len(overrides) == 1 + override = overrides[0] + assert override.startswith("+config_paths=[") + assert override.endswith("]") + assert "a.yaml" in override + assert "b.yaml" in override + + def test_config_is_prepended_before_passthrough_overrides(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "start", "--config", "a.yaml", "+foo=bar"]) + assert len(overrides) == 2 + assert "+config_paths=[a.yaml]" in overrides + assert "+foo=bar" in overrides + + def test_without_config_no_config_paths_added(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "start", "+foo=bar"]) + assert overrides == ["+foo=bar"] + + def test_config_rejected_on_non_config_command(self, monkeypatch: MonkeyPatch) -> None: + # `dataset rm` does not declare --config, so the router must reject it rather than leak it downstream. + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", "dataset", "rm", "--config", "x.yaml"]) + with pytest.raises(SystemExit): + main() + + +class TestStorageFlag: + @pytest.mark.parametrize( + "argv, expected_target", + [ + (["dataset", "upload"], "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli"), + (["dataset", "upload", "--storage", "hf"], "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli"), + (["dataset", "upload", "--storage", "gitlab"], "nemo_gym.cli.dataset:upload_jsonl_dataset_cli"), + (["dataset", "download"], "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli"), + (["dataset", "download", "--storage", "hf"], "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli"), + (["dataset", "download", "--storage", "gitlab"], "nemo_gym.cli.dataset:download_jsonl_dataset_cli"), + ], + ) + def test_storage_selects_backend(self, monkeypatch: MonkeyPatch, argv, expected_target) -> None: + target, _ = _dispatch_for(monkeypatch, argv) + assert target == expected_target + + def test_storage_does_not_leak_into_overrides(self, monkeypatch: MonkeyPatch) -> None: + # --storage only selects the target; it must not appear in the Hydra overrides. + _, overrides = _dispatch_for(monkeypatch, ["dataset", "upload", "--storage", "gitlab", "+foo=bar"]) + assert overrides == ["+foo=bar"] + + def test_invalid_storage_value_is_rejected(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr(sys, "argv", ["gym", "dataset", "upload", "--storage", "s3"]) + with pytest.raises(SystemExit): + main() + + +class TestEvalRunFlags: + @pytest.mark.parametrize( + "flag_argv, expected_override", + [ + (["--agent", "my_agent"], "+agent_name=my_agent"), + (["-a", "my_agent"], "+agent_name=my_agent"), + (["--input", "in.jsonl"], "+input_jsonl_fpath=in.jsonl"), + (["-i", "in.jsonl"], "+input_jsonl_fpath=in.jsonl"), + (["--output", "out.jsonl"], "+output_jsonl_fpath=out.jsonl"), + (["-o", "out.jsonl"], "+output_jsonl_fpath=out.jsonl"), + (["--limit", "1024"], "+limit=1024"), + (["--num-repeats", "4"], "+num_repeats=4"), + (["--concurrency", "10"], "+num_samples_in_parallel=10"), + (["--prompt-config", "p.yaml"], "+prompt_config=p.yaml"), + (["--split", "benchmark"], "+split=benchmark"), + (["--model", "openai/gpt-oss-120b"], "+policy_model_name=openai/gpt-oss-120b"), + (["-m", "openai/gpt-oss-120b"], "+policy_model_name=openai/gpt-oss-120b"), + (["--model-url", "http://0.0.0.0:10240/v1"], "+policy_base_url=http://0.0.0.0:10240/v1"), + (["--model-api-key", "sk-your-api-key"], "+policy_api_key=sk-your-api-key"), + (["--temperature", "1.0"], "+responses_create_params.temperature=1.0"), + (["--top-p", "1.0"], "+responses_create_params.top_p=1.0"), + (["--max-output-tokens", "4096"], "+responses_create_params.max_output_tokens=4096"), + (["--resume"], "+resume_from_cache=true"), + ], + ) + def test_flag_maps_to_single_override(self, monkeypatch: MonkeyPatch, flag_argv, expected_override) -> None: + _, overrides = _dispatch_for(monkeypatch, ["eval", "run", *flag_argv]) + assert overrides == [expected_override] + + def test_unset_flags_contribute_nothing(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--agent", "x"]) + assert overrides == ["+agent_name=x"] + + def test_default_dispatches_e2e(self, monkeypatch: MonkeyPatch) -> None: + target, _ = _dispatch_for(monkeypatch, ["eval", "run"]) + assert target == "nemo_gym.cli.eval:e2e_rollout_collection" + + def test_no_serve_dispatches_collect_without_override(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--no-serve"]) + assert target == "nemo_gym.cli.eval:collect_rollouts" + assert overrides == [] + + def test_readme_collect_rollouts_example(self, monkeypatch: MonkeyPatch) -> None: + # From resources_servers/my_weather_tool README: + # ng_collect_rollouts +agent_name=... +input_jsonl_fpath=... +output_jsonl_fpath=... +limit=1024 +num_repeats=1 + target, overrides = _dispatch_for( + monkeypatch, + [ + "eval", + "run", + "--no-serve", + "--agent", + "my_weather_tool_simple_agent", + "--input", + "resources_servers/my_weather_tool/data/example.jsonl", + "--output", + "resources_servers/my_weather_tool/data/example_rollouts.jsonl", + "--limit", + "1024", + "--num-repeats", + "1", + ], + ) + assert target == "nemo_gym.cli.eval:collect_rollouts" + assert set(overrides) == { + "+agent_name=my_weather_tool_simple_agent", + "+input_jsonl_fpath=resources_servers/my_weather_tool/data/example.jsonl", + "+output_jsonl_fpath=resources_servers/my_weather_tool/data/example_rollouts.jsonl", + "+limit=1024", + "+num_repeats=1", + } + + def test_readme_model_and_sampling_example(self, monkeypatch: MonkeyPatch) -> None: + # From the gpt-oss eval example: ++policy_* and ++responses_create_params.* overrides. + _, overrides = _dispatch_for( + monkeypatch, + [ + "eval", + "run", + "--model", + "openai/gpt-oss-120b", + "--model-url", + "http://0.0.0.0:10240/v1", + "--model-api-key", + "dummy_key", + "--temperature", + "1.0", + "--top-p", + "1.0", + ], + ) + assert set(overrides) == { + "+policy_model_name=openai/gpt-oss-120b", + "+policy_base_url=http://0.0.0.0:10240/v1", + "+policy_api_key=dummy_key", + "+responses_create_params.temperature=1.0", + "+responses_create_params.top_p=1.0", + } + + def test_flags_compose_with_config_and_passthrough(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + [ + "eval", + "run", + "--no-serve", + "--config", + "b.yaml", + "--agent", + "a", + "+responses_create_params.tool_choice=auto", + ], + ) + assert target == "nemo_gym.cli.eval:collect_rollouts" + assert "+config_paths=[b.yaml]" in overrides + assert "+agent_name=a" in overrides + assert "+responses_create_params.tool_choice=auto" in overrides # unknown +override passes through + + +class TestEnvTestResourceServerFlag: + def test_no_resource_server_runs_all(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["env", "test"]) + assert target == "nemo_gym.cli.env:test_all" + assert overrides == [] + + def test_resource_server_name_translates_to_entrypoint(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["env", "test", "--resources-server", "gpqa"]) + assert target == "nemo_gym.cli.env:test" + assert overrides == ["+entrypoint=resources_servers/gpqa"] + + def test_direct_entrypoint_override_also_runs_single(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["env", "test", "+entrypoint=resources_servers/gpqa"]) + assert target == "nemo_gym.cli.env:test" + assert overrides == ["+entrypoint=resources_servers/gpqa"] + + +class TestDatasetFlags: + def test_upload_hf_default(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + ["dataset", "upload", "-i", "data/train.jsonl", "--name", "my_ds", "--split", "train", "--create-pr"], + ) + assert target == "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_cli" + assert set(overrides) == { + "+input_jsonl_fpath=data/train.jsonl", + "+dataset_name=my_ds", + "+split=train", + "+create_pr=true", + } + + def test_upload_gitlab(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + [ + "dataset", + "upload", + "--storage", + "gitlab", + "-i", + "data/train.jsonl", + "--name", + "my_ds", + "--revision", + "0.0.1", + ], + ) + assert target == "nemo_gym.cli.dataset:upload_jsonl_dataset_cli" + overrides.remove( + "+revision=0.0.1" + ) # we set both version and revision because GitLab and HF use different keys + assert set(overrides) == { + "+input_jsonl_fpath=data/train.jsonl", + "+dataset_name=my_ds", + "+version=0.0.1", + } + + def test_download_hf_default(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + [ + "dataset", + "download", + "--repo-id", + "org/my_ds", + "--artifact", + "train.jsonl", + "--output-dir", + "./data", + "--split", + "train", + ], + ) + assert target == "nemo_gym.cli.dataset:download_jsonl_dataset_from_hf_cli" + assert set(overrides) == { + "+repo_id=org/my_ds", + "+artifact_fpath=train.jsonl", + "+output_dirpath=./data", + "+split=train", + } + + def test_download_gitlab(self, monkeypatch: MonkeyPatch) -> None: + # On download, --revision is GitLab-only and maps to +version (HF download has no revision field). + target, overrides = _dispatch_for( + monkeypatch, + [ + "dataset", + "download", + "--storage", + "gitlab", + "--name", + "my_ds", + "--revision", + "0.0.1", + "--artifact", + "train.jsonl", + "-o", + "./train.jsonl", + ], + ) + assert target == "nemo_gym.cli.dataset:download_jsonl_dataset_cli" + assert set(overrides) == { + "+dataset_name=my_ds", + "+version=0.0.1", + "+artifact_fpath=train.jsonl", + "+output_fpath=./train.jsonl", + } + + def test_rm(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["dataset", "rm", "--name", "my_ds"]) + assert target == "nemo_gym.cli.dataset:delete_jsonl_dataset_from_gitlab_cli" + assert overrides == ["+dataset_name=my_ds"] + + def test_migrate_revision_maps_to_hf_revision(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + ["dataset", "migrate", "-i", "data/train.jsonl", "--name", "my_ds", "--revision", "r1", "--create-pr"], + ) + assert target == "nemo_gym.cli.dataset:upload_jsonl_dataset_to_hf_and_delete_gitlab_cli" + assert set(overrides) == { + "+input_jsonl_fpath=data/train.jsonl", + "+dataset_name=my_ds", + "+revision=r1", + "+create_pr=true", + } + + def test_render(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, ["dataset", "render", "-i", "raw.jsonl", "--prompt-config", "p.yaml", "-o", "out.jsonl"] + ) + assert target == "nemo_gym.cli.dataset:materialize_prompts_cli" + assert set(overrides) == { + "+input_jsonl_fpath=raw.jsonl", + "+prompt_config=p.yaml", + "+output_jsonl_fpath=out.jsonl", + } + + def test_collate(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + [ + "dataset", + "collate", + "--config", + "c.yaml", + "--mode", + "train_preparation", + "--output-dir", + "./prep", + "--download", + ], + ) + assert target == "nemo_gym.cli.dataset:prepare_data" + assert set(overrides) == { + "+config_paths=[c.yaml]", + "+mode=train_preparation", + "+output_dirpath=./prep", + "+should_download=true", + } + + def test_bool_flags_omitted_when_unset(self, monkeypatch: MonkeyPatch) -> None: + # --create-pr not passed -> no +create_pr override leaks in. + _, overrides = _dispatch_for(monkeypatch, ["dataset", "upload", "--name", "my_ds"]) + assert overrides == ["+dataset_name=my_ds"] + + def test_collate_mode_rejects_invalid_choice(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr(sys, "argv", ["gym", "dataset", "collate", "--mode", "bogus"]) + with pytest.raises(SystemExit): + main() + + +class TestEvalAggregateFlags: + def test_aggregate_flags(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, ["eval", "aggregate", "-i", "results/rollouts-*.jsonl", "-o", "out.jsonl"] + ) + assert target == "nemo_gym.cli.eval:aggregate_rollouts" + assert set(overrides) == {"+input_glob=results/rollouts-*.jsonl", "+output_jsonl_fpath=out.jsonl"} + + +class TestDispatch: + """Exercise the real `dispatch` (every router test above stubs it), so argv rewriting and + module/attribute resolution are actually covered.""" + + def test_dispatch_imports_target_and_rewrites_argv(self, monkeypatch: MonkeyPatch) -> None: + # Direct unit test of dispatch (no parser): it splits "module:func", imports the module, + # resolves the attribute, rewrites sys.argv to argv[0] + overrides, then calls the target. + # A synthetic module registered in sys.modules keeps this decoupled from any real target. + captured: dict = {} + + def recorder() -> None: + captured["argv"] = list(sys.argv) + + fake_module = types.ModuleType("my_module.my_submodule") + fake_module.my_function = recorder + monkeypatch.setitem(sys.modules, "my_module.my_submodule", fake_module) + # argv starts with the parsed command tokens; dispatch must drop them but keep argv[0]. + monkeypatch.setattr(sys, "argv", ["my_command", "subcommand", "--some-flag", "some-value"]) + cli_main.dispatch("my_module.my_submodule:my_function", ["+a=1", "+b=2"]) + assert captured["argv"] == ["my_command", "+a=1", "+b=2"] + + def test_main_to_dispatch_end_to_end(self, monkeypatch: MonkeyPatch) -> None: + # Drive dispatch from main() with only the leaf target stubbed: the target string is split, + # the module is imported, the attribute is resolved, and sys.argv is rewritten for real. + # Mix `--flag` asset selectors (translated and coalesced into +config_paths) with raw + # `+`/`++` Hydra passthroughs. + captured: dict = {} + + def recorder() -> None: + captured["argv"] = list(sys.argv) + + monkeypatch.setattr("nemo_gym.cli.eval.e2e_rollout_collection", recorder) + monkeypatch.setattr( + sys, + "argv", + [ + "gym", + "eval", + "run", + "--benchmark", + "aime24", + "--model-type", + "openai_model", + "++responses_create_params.reasoning.effort=low", + "+wandb_project=gym-dev", + ], + ) + main() + # dispatch drops the parsed command tokens, keeping argv[0] and the resolved overrides. + assert captured["argv"][0] == "gym" + config_paths, others = _split_overrides(captured["argv"][1:]) + assert any(p.endswith("benchmarks/aime24/config.yaml") for p in config_paths) + assert any(p.endswith("responses_api_models/openai_model/configs/openai_model.yaml") for p in config_paths) + assert others == {"++responses_create_params.reasoning.effort=low", "+wandb_project=gym-dev"} + + def test_dispatch_raises_on_unresolvable_attribute(self, monkeypatch: MonkeyPatch) -> None: + # A typo in a Command target must surface as a resolution error, not silently no-op. + monkeypatch.setattr(sys, "argv", ["gym"]) + with pytest.raises(AttributeError): + cli_main.dispatch("nemo_gym.cli.eval:no_such_function", []) + + +class TestEvalProfileFlags: + def test_profile_flags(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, ["eval", "profile", "--inputs", "in.jsonl", "--rollouts", "r.jsonl"] + ) + assert target == "nemo_gym.cli.eval:reward_profile" + assert set(overrides) == { + "+materialized_inputs_jsonl_fpath=in.jsonl", + "+rollouts_jsonl_fpath=r.jsonl", + } + + def test_profile_does_not_accept_config(self, monkeypatch: MonkeyPatch) -> None: + # reward_profile reads file paths, not config_paths, so --config is not offered and is rejected. + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", "eval", "profile", "--config", "x.yaml"]) + with pytest.raises(SystemExit): + main() + + +class TestEnvRunFlags: + def test_model_flags(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for( + monkeypatch, + [ + "env", + "start", + "--config", + "c.yaml", + "--model", + "gpt", + "--model-url", + "http://x", + "--model-api-key", + "k", + ], + ) + assert target == "nemo_gym.cli.env:run" + assert set(overrides) == { + "+config_paths=[c.yaml]", + "+policy_model_name=gpt", + "+policy_base_url=http://x", + "+policy_api_key=k", + } + + +class TestModelFlag: + """`--model` is the single served-model identifier (name, HF id, or local checkpoint path) for any backend.""" + + def test_local_vllm_deployment_flow(self, monkeypatch: MonkeyPatch) -> None: + # The deployment invocation: select the local vLLM server type and pass the checkpoint to serve via --model. + _, overrides = _dispatch_for( + monkeypatch, + ["eval", "run", "--model-type", "local_vllm_model", "--model", "Qwen/Qwen3-8B"], + ) + paths, others = _split_overrides(overrides) + assert paths == {str(WORKING_DIR / "responses_api_models/local_vllm_model/configs/local_vllm_model.yaml")} + assert others == {"+policy_model_name=Qwen/Qwen3-8B"} + + def test_short_alias_on_env_run(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "start", "-m", "/ckpt/path"]) + assert overrides == ["+policy_model_name=/ckpt/path"] + + +class TestEnvInitFlags: + def test_resource_server_translates_to_entrypoint(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["env", "init", "--resources-server", "my_server"]) + assert target == "nemo_gym.cli.env:init_resources_server" + assert overrides == ["+entrypoint=resources_servers/my_server"] + + +class TestEnvPackagesFlags: + def test_flags(self, monkeypatch: MonkeyPatch) -> None: + target, overrides = _dispatch_for(monkeypatch, ["env", "packages", "--resources-server", "gpqa", "--outdated"]) + assert target == "nemo_gym.cli.env:pip_list" + assert set(overrides) == { + "+entrypoint=resources_servers/gpqa", + "+outdated=true", + } + + +class TestJsonFlag: + @pytest.mark.parametrize( + "argv, expected_target", + [ + (["list", "benchmarks", "--json"], "nemo_gym.cli.eval:list_benchmarks"), + (["env", "status", "--json"], "nemo_gym.cli.env:status"), + ], + ) + def test_json_becomes_config_override(self, monkeypatch: MonkeyPatch, argv, expected_target) -> None: + # Reporting commands surface --json as the reserved `json` config key, read centrally by cli.output.emit. + target, overrides = _dispatch_for(monkeypatch, argv) + assert target == expected_target + assert overrides == ["+json=true"] + + def test_no_json_no_override(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["list", "benchmarks"]) + assert overrides == [] + + +class TestSearch: + def test_search_routes_to_list_with_query(self, monkeypatch: MonkeyPatch) -> None: + # `gym search ` reuses the benchmarks listing, passing the query as the `query` config key. + target, overrides = _dispatch_for(monkeypatch, ["search", "math"]) + assert target == "nemo_gym.cli.eval:list_benchmarks" + assert overrides == ["+query=math"] + + def test_search_json(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["search", "math", "--json"]) + assert set(overrides) == {"+query=math", "+json=true"} + + def test_version_json_dispatches_with_override(self, monkeypatch: MonkeyPatch) -> None: + # `gym --version --json` is the top-level path; it still forwards +json=true to the version command. + target, overrides = _dispatch_for(monkeypatch, ["--version", "--json"]) + assert target == "nemo_gym.cli.general:version" + assert overrides == ["+json=true"] + + def test_env_packages_json_maps_to_uv_format(self, monkeypatch: MonkeyPatch) -> None: + # env packages delegates to `uv pip list`, so --json maps onto its own --format=json rather than +json=true. + target, overrides = _dispatch_for(monkeypatch, ["env", "packages", "--resources-server", "mcqa", "--json"]) + assert target == "nemo_gym.cli.env:pip_list" + assert set(overrides) == {"+entrypoint=resources_servers/mcqa", "+format=json"} + + def test_env_packages_without_json(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "packages", "--resources-server", "mcqa"]) + assert overrides == ["+entrypoint=resources_servers/mcqa"] + + +class TestVerboseFlag: + @pytest.mark.parametrize("flag", ["-v", "--verbose"]) + def test_verbose_injects_config_override(self, monkeypatch: MonkeyPatch, flag: str) -> None: + # --verbose flows through the config (so it reaches servers), not just the local logger. + _, overrides = _dispatch_for(monkeypatch, ["env", "status", flag]) + assert overrides == ["+verbose=true"] + + def test_no_verbose_no_override(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["env", "status"]) + assert overrides == [] + + def test_verbose_prepended_before_other_overrides(self, monkeypatch: MonkeyPatch) -> None: + _, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--verbose", "--agent", "a", "+x=1"]) + assert "+verbose=true" in overrides + assert "+agent_name=a" in overrides + assert "+x=1" in overrides + + def test_config_verbose_sets_debug_on_load(self, monkeypatch: MonkeyPatch) -> None: + # The server-side path: a config carrying `verbose` (forwarded via env var) raises the log level. + monkeypatch.setattr(gc, "_GLOBAL_CONFIG_DICT", None) + monkeypatch.setenv(NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, "verbose: true\nsome_server: {}\n") + root = logging.getLogger() + original = root.level + try: + root.setLevel(logging.WARNING) + gc.get_global_config_dict() + assert root.level == logging.DEBUG + finally: + root.setLevel(original) + + def test_config_without_verbose_keeps_level(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr(gc, "_GLOBAL_CONFIG_DICT", None) + monkeypatch.setenv(NEMO_GYM_CONFIG_DICT_ENV_VAR_NAME, "some_server: {}\n") + root = logging.getLogger() + original = root.level + try: + root.setLevel(logging.WARNING) + gc.get_global_config_dict() + assert root.level == logging.WARNING + finally: + root.setLevel(original) + + +class TestAssetSelectors: + """Named selectors (--benchmark, --resources-server, --model-type) that resolve a name to a default config path. + + Each example mirrors a real invocation from the docs/READMEs, so the sugar stays faithful to the documented + config paths it replaces. The legacy `+config_paths=[...]` form each one is derived from is cited inline. + """ + + @pytest.mark.parametrize( + "argv, expected_config", + [ + # benchmarks/gsm8k/README.md: ng_prepare_benchmark "+config_paths=[benchmarks/gsm8k/config.yaml]" + (["eval", "prepare", "--benchmark", "gsm8k"], "benchmarks/gsm8k/config.yaml"), + # benchmarks/aime25-x/README.md: ng_prepare_benchmark "+config_paths=[benchmarks/aime25-x/config.yaml]" + (["eval", "prepare", "--benchmark", "aime25-x"], "benchmarks/aime25-x/config.yaml"), + # benchmarks/gpqa/README.md: ng_run "+config_paths=[benchmarks/gpqa/config.yaml]" (start a benchmark's servers) + (["env", "start", "--benchmark", "gpqa"], "benchmarks/gpqa/config.yaml"), + # README.md / quickstart.mdx: resources_servers/mcqa/configs/mcqa.yaml + (["env", "start", "--resources-server", "mcqa"], "resources_servers/mcqa/configs/mcqa.yaml"), + # model-server/vllm.mdx: resources_servers/example_multi_step/configs/example_multi_step.yaml + ( + ["env", "start", "--resources-server", "example_multi_step"], + "resources_servers/example_multi_step/configs/example_multi_step.yaml", + ), + # README.md / quickstart.mdx: responses_api_models/openai_model/configs/openai_model.yaml + ( + ["env", "start", "--model-type", "openai_model"], + "responses_api_models/openai_model/configs/openai_model.yaml", + ), + # model-server/vllm.mdx: responses_api_models/vllm_model/configs/vllm_model.yaml + ( + ["env", "start", "--model-type", "vllm_model"], + "responses_api_models/vllm_model/configs/vllm_model.yaml", + ), + ], + ) + def test_name_resolves_to_config_path(self, monkeypatch: MonkeyPatch, argv, expected_config) -> None: + _, overrides = _dispatch_for(monkeypatch, argv) + assert overrides == [f"+config_paths=[{WORKING_DIR / (expected_config)}]"] + + def test_quickstart_resource_server_plus_model(self, monkeypatch: MonkeyPatch) -> None: + # README.md / quickstart.mdx: + # ng_run "+config_paths=[resources_servers/mcqa/configs/mcqa.yaml, + # responses_api_models/openai_model/configs/openai_model.yaml]" + target, overrides = _dispatch_for( + monkeypatch, ["env", "start", "--resources-server", "mcqa", "--model-type", "openai_model"] + ) + assert target == "nemo_gym.cli.env:run" + paths, others = _split_overrides(overrides) + assert paths == { + str(WORKING_DIR / "resources_servers/mcqa/configs/mcqa.yaml"), + str(WORKING_DIR / "responses_api_models/openai_model/configs/openai_model.yaml"), + } + assert others == set() + + def test_gpqa_benchmark_plus_model(self, monkeypatch: MonkeyPatch) -> None: + # benchmarks/gpqa/README.md: + # ng_run "+config_paths=[benchmarks/gpqa/config.yaml,responses_api_models/vllm_model/configs/vllm_model.yaml]" + _, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--benchmark", "gpqa", "--model-type", "vllm_model"]) + paths, others = _split_overrides(overrides) + assert paths == { + str(WORKING_DIR / "benchmarks/gpqa/config.yaml"), + str(WORKING_DIR / "responses_api_models/vllm_model/configs/vllm_model.yaml"), + } + assert others == set() + + def test_cli_reference_e2e_rollout_example(self, monkeypatch: MonkeyPatch) -> None: + # fern .../reference/cli-commands.mdx ng_e2e_collect_rollouts example: + # config_paths="responses_api_models/openai_model/configs/openai_model.yaml, + # resources_servers/math_with_judge/configs/math_with_judge.yaml" + # ng_e2e_collect_rollouts "+config_paths=[$config_paths]" + # ++output_jsonl_fpath=results/test_e2e_rollout_collection/aime24.jsonl ++split=validation + target, overrides = _dispatch_for( + monkeypatch, + [ + "eval", + "run", + "--resources-server", + "math_with_judge", + "--model-type", + "openai_model", + "--output", + "results/test_e2e_rollout_collection/aime24.jsonl", + "--split", + "validation", + ], + ) + assert target == "nemo_gym.cli.eval:e2e_rollout_collection" + paths, others = _split_overrides(overrides) + assert paths == { + str(WORKING_DIR / "resources_servers/math_with_judge/configs/math_with_judge.yaml"), + str(WORKING_DIR / "responses_api_models/openai_model/configs/openai_model.yaml"), + } + assert others == { + "+output_jsonl_fpath=results/test_e2e_rollout_collection/aime24.jsonl", + "+split=validation", + } + + def test_cli_reference_prepare_data_example(self, monkeypatch: MonkeyPatch) -> None: + # fern .../reference/cli-commands.mdx ng_prepare_data example: + # config_paths includes resources_servers/example_multi_step/configs/example_multi_step.yaml + # ng_prepare_data "+config_paths=[...]" +output_dirpath=data/example_multi_step +mode=example_validation + target, overrides = _dispatch_for( + monkeypatch, + [ + "dataset", + "collate", + "--resources-server", + "example_multi_step", + "--mode", + "example_validation", + "--output-dir", + "data/example_multi_step", + ], + ) + assert target == "nemo_gym.cli.dataset:prepare_data" + paths, others = _split_overrides(overrides) + assert paths == {str(WORKING_DIR / "resources_servers/example_multi_step/configs/example_multi_step.yaml")} + assert others == { + "+mode=example_validation", + "+output_dirpath=data/example_multi_step", + } + + def test_resource_server_flavor_syntax(self, monkeypatch: MonkeyPatch) -> None: + # `/` picks a named config inside the server's configs/ dir; math_with_judge ships several + # flavoured configs (see reference/faq.mdx, which pairs a math_with_judge dataset flavour for profiling). + _, overrides = _dispatch_for(monkeypatch, ["eval", "run", "--resources-server", "math_with_judge/dapo17k"]) + assert overrides == [ + f"+config_paths=[{WORKING_DIR / 'resources_servers/math_with_judge/configs/dapo17k.yaml'}]" + ] + + def test_benchmark_flavor_syntax(self, monkeypatch: MonkeyPatch) -> None: + # Benchmarks are flavoured too: flavor is a sibling `.yaml` (no configs/ dir), default `config`. + # e.g. benchmarks/finance_sec_search ships config_web_search.yaml alongside the default config.yaml. + _, overrides = _dispatch_for( + monkeypatch, ["eval", "prepare", "--benchmark", "finance_sec_search/config_web_search"] + ) + assert overrides == [f"+config_paths=[{WORKING_DIR / 'benchmarks/finance_sec_search/config_web_search.yaml'}]"] + + def test_environment_selector_resolves_to_config(self, monkeypatch: MonkeyPatch) -> None: + # Environments mirror benchmarks: `--environment ` loads `environments//config.yaml`. + # Used by `gym env start` / `gym eval run` for environments/ (see environments/*/README.md). + _, overrides = _dispatch_for(monkeypatch, ["env", "start", "--environment", "circle_count"]) + assert overrides == [f"+config_paths=[{WORKING_DIR / 'environments/circle_count/config.yaml'}]"] + + def test_selectors_merge_into_single_config_paths(self, monkeypatch: MonkeyPatch) -> None: + # --config and multiple asset selectors all feed one +config_paths list (Hydra rejects duplicates). + # _split_overrides asserts they coalesce into a single token. + _, overrides = _dispatch_for( + monkeypatch, + ["eval", "run", "--config", "extra.yaml", "--resources-server", "mcqa", "--model-type", "openai_model"], + ) + paths, others = _split_overrides(overrides) + assert paths == { + "extra.yaml", # raw --config value passes through verbatim; only name selectors get rooted + str(WORKING_DIR / "resources_servers/mcqa/configs/mcqa.yaml"), + str(WORKING_DIR / "responses_api_models/openai_model/configs/openai_model.yaml"), + } + assert others == set() + + def test_unknown_benchmark_errors_with_available_hint(self, monkeypatch: MonkeyPatch, capsys) -> None: + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", "eval", "prepare", "--benchmark", "does_not_exist"]) + with pytest.raises(SystemExit): + main() + err = capsys.readouterr().err + assert "benchmarks/does_not_exist/config.yaml" in err + assert "does not exist" in err + assert "benchmarks/" in err + + def test_unknown_flavor_error_points_at_configs_dir(self, monkeypatch: MonkeyPatch, capsys) -> None: + # For a known server with an unknown flavor, the hint should point at that server's configs/ dir. + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", "env", "start", "--resources-server", "mcqa/nope"]) + with pytest.raises(SystemExit): + main() + err = capsys.readouterr().err + assert "resources_servers/mcqa/configs/nope.yaml" in err + assert "resources_servers/mcqa/configs/" in err + + +class TestDidYouMean: + """difflib-backed "did you mean?" hints for mistyped commands, flags, and component names (proposal UX 4).""" + + def test_helper_suggests_close_match(self) -> None: + assert cli_main._did_you_mean("evl", ["list", "eval", "env"]) == " Did you mean `eval`?" + + def test_helper_silent_when_nothing_close(self) -> None: + assert cli_main._did_you_mean("zzzzzz", ["list", "eval", "env"]) == "" + + def _run_expecting_exit(self, monkeypatch: MonkeyPatch, capsys, argv: list[str]) -> str: + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr(sys, "argv", ["gym", *argv]) + with pytest.raises(SystemExit): + main() + return capsys.readouterr().err + + def test_mistyped_group(self, monkeypatch: MonkeyPatch, capsys) -> None: + err = self._run_expecting_exit(monkeypatch, capsys, ["evl"]) + assert "invalid choice: 'evl'" in err + assert "Did you mean `eval`?" in err + + def test_mistyped_action(self, monkeypatch: MonkeyPatch, capsys) -> None: + err = self._run_expecting_exit(monkeypatch, capsys, ["eval", "rnu"]) + assert "Did you mean `run`?" in err + + def test_mistyped_flag_choice(self, monkeypatch: MonkeyPatch, capsys) -> None: + # --storage validates choices, so the parser-level hint kicks in. + err = self._run_expecting_exit(monkeypatch, capsys, ["dataset", "upload", "--storage", "gitlb"]) + assert "Did you mean `gitlab`?" in err + + def test_misspelled_flag(self, monkeypatch: MonkeyPatch, capsys) -> None: + err = self._run_expecting_exit(monkeypatch, capsys, ["eval", "run", "--benchmrk", "aalcr"]) + assert "unrecognized arguments: --benchmrk" in err + assert "Did you mean `--benchmark`?" in err + + def test_misspelled_component_name(self, monkeypatch: MonkeyPatch, capsys) -> None: + err = self._run_expecting_exit(monkeypatch, capsys, ["eval", "prepare", "--benchmark", "aalcrr"]) + assert "Did you mean `aalcr`?" in err + + def test_misspelled_component_flavor(self, monkeypatch: MonkeyPatch, capsys) -> None: + err = self._run_expecting_exit( + monkeypatch, capsys, ["eval", "run", "--resources-server", "math_with_judge/dapo17"] + ) + assert "Did you mean `dapo17k`?" in err + + +class TestSearchDir: + """--search-dir registers extra roots that the name->config selectors also search (REQ 5).""" + + def _make_user_benchmark(self, tmp_path, name: str = "mybench") -> None: + bench_dir = tmp_path / "benchmarks" / name + bench_dir.mkdir(parents=True) + (bench_dir / "config.yaml").write_text("{}\n") + + def test_resolves_component_from_user_dir(self, monkeypatch: MonkeyPatch, tmp_path) -> None: + self._make_user_benchmark(tmp_path) + _, overrides = _dispatch_for( + monkeypatch, ["eval", "prepare", "--benchmark", "mybench", "--search-dir", str(tmp_path)] + ) + # User-dir matches are returned rooted so Hydra can resolve them. + assert overrides == [f"+config_paths=[{tmp_path / 'benchmarks' / 'mybench' / 'config.yaml'}]"] + + def test_builtin_resolves_when_search_dir_lacks_it(self, monkeypatch: MonkeyPatch, tmp_path) -> None: + # A built-in still resolves under WORKING_DIR when a --search-dir is provided that does not shadow it. + _, overrides = _dispatch_for( + monkeypatch, ["eval", "prepare", "--benchmark", "gsm8k", "--search-dir", str(tmp_path)] + ) + assert overrides == [f"+config_paths=[{WORKING_DIR / 'benchmarks/gsm8k/config.yaml'}]"] + + def test_ambiguous_match_errors(self, monkeypatch: MonkeyPatch, tmp_path, capsys) -> None: + # A built-in name also present in a --search-dir is ambiguous; the user must disambiguate with --config. + self._make_user_benchmark(tmp_path, name="gsm8k") # gsm8k also exists under WORKING_DIR + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr( + sys, "argv", ["gym", "eval", "prepare", "--benchmark", "gsm8k", "--search-dir", str(tmp_path)] + ) + with pytest.raises(SystemExit): + main() + err = capsys.readouterr().err + assert "ambiguous" in err + assert str(WORKING_DIR / "benchmarks" / "gsm8k" / "config.yaml") in err + assert str(tmp_path / "benchmarks" / "gsm8k" / "config.yaml") in err + + def test_search_dir_alone_emits_nothing(self, monkeypatch: MonkeyPatch, tmp_path) -> None: + # --search-dir is consumed by the selectors; on its own it is not a Hydra override. + _, overrides = _dispatch_for(monkeypatch, ["eval", "prepare", "--search-dir", str(tmp_path)]) + assert overrides == [] + + def test_did_you_mean_spans_user_dir(self, monkeypatch: MonkeyPatch, tmp_path, capsys) -> None: + self._make_user_benchmark(tmp_path) + monkeypatch.setattr(cli_main, "dispatch", lambda target, overrides: None) + monkeypatch.setattr( + sys, "argv", ["gym", "eval", "prepare", "--benchmark", "mybnch", "--search-dir", str(tmp_path)] + ) + with pytest.raises(SystemExit): + main() + assert "Did you mean `mybench`?" in capsys.readouterr().err diff --git a/tests/unit_tests/test_cli_setup_command.py b/tests/unit_tests/test_cli_setup_command.py index b751c4472d..4f531eedc1 100644 --- a/tests/unit_tests/test_cli_setup_command.py +++ b/tests/unit_tests/test_cli_setup_command.py @@ -19,8 +19,8 @@ import pytest from pytest import MonkeyPatch, raises -import nemo_gym.cli_setup_command -from nemo_gym.cli_setup_command import ( +import nemo_gym.cli.setup_command +from nemo_gym.cli.setup_command import ( _get_nemo_gym_install_flags, _get_nemo_gym_version_spec, run_command, @@ -243,15 +243,15 @@ def test_uv_venv_dir_and_skip_install_when_venv_present(self, tmp_path: Path) -> class TestCLISetupCommandRunCommand: def _setup(self, monkeypatch: MonkeyPatch) -> tuple[MagicMock, MagicMock]: Popen_mock = MagicMock() - monkeypatch.setattr(nemo_gym.cli_setup_command, "Popen", Popen_mock) + monkeypatch.setattr(nemo_gym.cli.setup_command, "Popen", Popen_mock) get_global_config_dict_mock = MagicMock(return_value={"uv_cache_dir": "default uv cache dir"}) - monkeypatch.setattr(nemo_gym.cli_setup_command, "get_global_config_dict", get_global_config_dict_mock) + monkeypatch.setattr(nemo_gym.cli.setup_command, "get_global_config_dict", get_global_config_dict_mock) - monkeypatch.setattr(nemo_gym.cli_setup_command, "environ", dict()) + monkeypatch.setattr(nemo_gym.cli.setup_command, "environ", dict()) - monkeypatch.setattr(nemo_gym.cli_setup_command, "stdout", "stdout") - monkeypatch.setattr(nemo_gym.cli_setup_command, "stderr", "stderr") + monkeypatch.setattr(nemo_gym.cli.setup_command, "stdout", "stdout") + monkeypatch.setattr(nemo_gym.cli.setup_command, "stderr", "stderr") return Popen_mock, get_global_config_dict_mock @@ -276,7 +276,7 @@ def test_sanity(self, monkeypatch: MonkeyPatch) -> None: def test_custom_pythonpath(self, monkeypatch: MonkeyPatch) -> None: Popen_mock, get_global_config_dict_mock = self._setup(monkeypatch) - monkeypatch.setattr(nemo_gym.cli_setup_command, "environ", {"PYTHONPATH": "existing pythonpath"}) + monkeypatch.setattr(nemo_gym.cli.setup_command, "environ", {"PYTHONPATH": "existing pythonpath"}) run_command( command="my command", diff --git a/tests/unit_tests/test_server_status.py b/tests/unit_tests/test_server_status.py index 609ddbeae9..2a17e8cad1 100644 --- a/tests/unit_tests/test_server_status.py +++ b/tests/unit_tests/test_server_status.py @@ -12,15 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging from io import StringIO from unittest.mock import MagicMock import requests from pytest import MonkeyPatch -from nemo_gym.cli import ServerInstanceDisplayConfig from nemo_gym.server_status import StatusCommand -from nemo_gym.server_utils import ServerClient +from nemo_gym.server_utils import ServerClient, ServerInstanceDisplayConfig class TestServerStatus: @@ -209,7 +209,7 @@ def test_discover_servers(self, monkeypatch: MonkeyPatch) -> None: first_call = mock_get.call_args_list[0] assert first_call[0][0] == "http://127.0.0.1:11000/server_instances" - def test_discover_servers_head_server_down(self, monkeypatch: MonkeyPatch, capsys) -> None: + def test_discover_servers_head_server_down(self, monkeypatch: MonkeyPatch, capsys, caplog) -> None: mock_head_config = MagicMock() mock_head_config.host = "127.0.0.1" mock_head_config.port = 11000 @@ -220,9 +220,11 @@ def test_discover_servers_head_server_down(self, monkeypatch: MonkeyPatch, capsy monkeypatch.setattr(requests, "get", mock_get) cmd = StatusCommand() - servers = cmd.discover_servers() + with caplog.at_level(logging.WARNING): + servers = cmd.discover_servers() assert len(servers) == 0 - captured = capsys.readouterr() - assert "Could not connect to head server" in captured.out - assert "ng_run" in captured.out + # The warning goes through logging (stderr), keeping stdout machine-readable for `--json`. + assert capsys.readouterr().out == "" + assert "Could not connect to head server" in caplog.text + assert "gym env start" in caplog.text