Clarify agent skill handling (#886)

Author: willccbb

docs/training.md

@@ -99,7 +99,7 @@ This will clone and install the `prime-rl` trainer and its dependencies, and set
 
 Then, you can start training with:
 ```bash
-uv run prime-rl @ configs/prime-rl/wiki-search.toml
+uv run prime-rl configs/prime-rl/wiki-search.toml
 ```
 
 This will launch a tmux session with separate panes for the trainer, orchestrator, and inference server. For further configuration options, see the [prime-rl documentation](https://docs.primeintellect.ai/prime-rl). 

skills/create-environments/SKILL.md

@@ -99,6 +99,13 @@ prime env push --path ./environments/my_env --visibility PRIVATE
 prime eval run owner/my-env -m gpt-4.1-mini -n 200 -r 3 -s
 ```
 
+## Synthetic Data
+1. Ask users for preferences on which LLMs to use for synthetic data generation and curation before implementation.
+2. Prefer generating synthetic data from raw source documents whenever possible instead of relying only on hand-authored prompts.
+3. Use LLM orchestration (planner/generator/validator loops) to improve sample quality and diversity.
+4. Use back-translation: start from complete materials and decompose them into incomplete tasks, criteria, or partial artifacts that the model must reconstruct.
+5. Use fan-out subtopic sampling from LLMs to expand coverage and avoid overfitting to a narrow slice of the domain.
+
 ## Deliverable Format
 Report:
 1. Environment ID and path.

skills/train-with-environments/SKILL.md

@@ -9,16 +9,18 @@ description: Train models with verifiers environments using hosted RL or prime-r
 Run stable RL training loops with environment-aware hyperparameter choices and clear diagnostics.
 
 ## Preferred Training Paths
-1. Hosted Training service path from lab setup:
+1. By default, assume users intend to use Hosted Training unless they explicitly ask for self-managed training.
+2. Hosted Training service path from lab setup:
 ```bash
 prime lab setup
 ```
-2. Self-managed `prime-rl` workflow:
+3. Self-managed `prime-rl` workflow:
 ```bash
 prime lab setup --prime-rl
-uv run prime-rl @ configs/prime-rl/wiki-search.toml
+uv run prime-rl configs/prime-rl/wiki-search.toml
 ```
-3. Runtime expectation:
+4. Treat `prime-rl` as a power-user path and assume users are comfortable working with GPU infrastructure and troubleshooting.
+5. Runtime expectation:
 - Hosted Training is intended to be launched from a CPU machine.
 - Local `prime-rl` training requires local GPU access.
 

tests/test_setup_script.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import json
+import os
 from pathlib import Path
 
 from verifiers.scripts import setup
@@ -101,3 +103,186 @@ def test_sync_prime_skills_creates_dot_prime_tree(tmp_path: Path, monkeypatch) -
             ".prime/skills/brainstorm/SKILL.md",
         ),
     ]
+
+
+def test_prepare_agent_skill_dirs_materializes_skills_from_prime(
+    tmp_path: Path, monkeypatch
+) -> None:
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(setup, "LAB_SKILLS", ["create-environments", "brainstorm"])
+
+    for skill_name in setup.LAB_SKILLS:
+        skill_dir = tmp_path / ".prime" / "skills" / skill_name
+        skill_dir.mkdir(parents=True, exist_ok=True)
+        (skill_dir / "SKILL.md").write_text(f"{skill_name}\n")
+
+    setup._prepare_agent_skill_dirs(["codex"])
+
+    for skill_name in setup.LAB_SKILLS:
+        source = tmp_path / ".prime" / "skills" / skill_name
+        target = tmp_path / ".codex" / "skills" / skill_name
+        assert target.exists()
+        assert (target / "SKILL.md").exists()
+        if target.is_symlink():
+            assert target.resolve() == source.resolve()
+
+
+def test_prepare_agent_skill_dirs_is_safe_with_existing_links(
+    tmp_path: Path, monkeypatch
+) -> None:
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(setup, "LAB_SKILLS", ["create-environments"])
+
+    source = tmp_path / ".prime" / "skills" / "create-environments"
+    source.mkdir(parents=True, exist_ok=True)
+    (source / "SKILL.md").write_text("create-environments\n")
+
+    target = tmp_path / ".codex" / "skills" / "create-environments"
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.symlink_to(
+        os.path.relpath(source, start=target.parent),
+        target_is_directory=True,
+    )
+
+    setup._prepare_agent_skill_dirs(["codex"])
+
+    assert target.is_symlink()
+    assert target.resolve() == source.resolve()
+    assert (target / "SKILL.md").exists()
+
+
+def test_prepare_agent_skill_dirs_uses_mapped_root_for_amp(
+    tmp_path: Path, monkeypatch
+) -> None:
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(setup, "LAB_SKILLS", ["create-environments"])
+
+    source = tmp_path / ".prime" / "skills" / "create-environments"
+    source.mkdir(parents=True, exist_ok=True)
+    (source / "SKILL.md").write_text("create-environments\n")
+
+    setup._prepare_agent_skill_dirs(["amp"])
+
+    assert (
+        tmp_path / ".agents" / "skills" / "create-environments" / "SKILL.md"
+    ).exists()
+    assert not (tmp_path / ".amp" / "skills").exists()
+
+
+def test_prepare_agent_skill_dirs_supports_skill_name_mapping(
+    tmp_path: Path, monkeypatch
+) -> None:
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(setup, "LAB_SKILLS", ["create-environments"])
+    monkeypatch.setattr(
+        setup,
+        "AGENT_SKILL_NAME_MAP",
+        {"amp": {"create-environments": "create-envs"}},
+    )
+
+    source = tmp_path / ".prime" / "skills" / "create-environments"
+    source.mkdir(parents=True, exist_ok=True)
+    (source / "SKILL.md").write_text("create-environments\n")
+
+    setup._prepare_agent_skill_dirs(["amp"])
+
+    assert (tmp_path / ".agents" / "skills" / "create-envs" / "SKILL.md").exists()
+    assert not (tmp_path / ".agents" / "skills" / "create-environments").exists()
+
+
+def test_run_setup_prints_post_setup_call_to_action(
+    tmp_path: Path, monkeypatch, capsys
+) -> None:
+    monkeypatch.chdir(tmp_path)
+
+    monkeypatch.setattr(setup.wget, "download", _fake_download_factory([]))
+    monkeypatch.setattr(setup, "download_configs", lambda *_: None)
+    monkeypatch.setattr(setup, "sync_prime_skills", lambda: None)
+
+    setup.run_setup(skip_install=True, skip_agents_md=True)
+
+    output = capsys.readouterr().out
+    assert "Prepared .codex/skills" in output
+    assert output.index("Prepared .codex/skills") < output.index("get started")
+    assert "get started" in output
+    assert "quick commands" in output
+    assert "ask codex" in output
+    assert "example prompt" not in output
+    assert 'ask codex: "I want to train a model for my task domain.' not in output
+    assert "idea -> environment -> eval -> training" in output
+    assert "prime env init my-env" in output
+    assert "prime env install my-env" not in output
+    assert "prime eval run my-env -m gpt-5-nano -n 5" in output
+    assert "prime eval tui" in output
+    assert "prime rl run configs/rl/wiki-search.toml" in output
+    assert "prime gepa run my-env -m gpt-5-nano" in output
+
+
+def test_run_setup_prints_prime_rl_post_setup_call_to_action(
+    tmp_path: Path, monkeypatch, capsys
+) -> None:
+    monkeypatch.chdir(tmp_path)
+
+    monkeypatch.setattr(setup.wget, "download", _fake_download_factory([]))
+    monkeypatch.setattr(setup, "download_configs", lambda *_: None)
+    monkeypatch.setattr(setup, "sync_prime_skills", lambda: None)
+    monkeypatch.setattr(setup, "install_prime_rl", lambda: None)
+    monkeypatch.setattr(setup, "install_environments_to_prime_rl", lambda: None)
+
+    setup.run_setup(skip_install=True, skip_agents_md=True, prime_rl=True)
+
+    output = capsys.readouterr().out
+    assert "get started" in output
+    assert "quick commands" in output
+    assert "ask codex" in output
+    assert "example prompt" not in output
+    assert "prime env install my-env" not in output
+    assert "uv run prime-rl configs/prime-rl/wiki-search.toml" in output
+    assert "prime rl run configs/rl/wiki-search.toml" not in output
+
+
+def test_run_setup_persists_lab_choices_metadata(tmp_path: Path, monkeypatch) -> None:
+    monkeypatch.chdir(tmp_path)
+
+    monkeypatch.setattr(setup.wget, "download", _fake_download_factory([]))
+    monkeypatch.setattr(setup, "download_configs", lambda *_: None)
+    monkeypatch.setattr(setup, "sync_prime_skills", lambda: None)
+
+    setup.run_setup(
+        skip_install=True,
+        skip_agents_md=True,
+        agents="codex,cursor,codex",
+        no_interactive=True,
+    )
+
+    metadata_path = tmp_path / ".prime" / "lab.json"
+    assert metadata_path.exists()
+    metadata = json.loads(metadata_path.read_text())
+    assert metadata["setup_source"] == "prime lab setup"
+    assert metadata["choices"] == {
+        "agents": ["codex", "cursor"],
+        "primary_agent": "codex",
+        "use_multiple_agents": True,
+    }
+
+
+def test_run_setup_persists_default_lab_choices_metadata(
+    tmp_path: Path, monkeypatch
+) -> None:
+    monkeypatch.chdir(tmp_path)
+
+    monkeypatch.setattr(setup.wget, "download", _fake_download_factory([]))
+    monkeypatch.setattr(setup, "download_configs", lambda *_: None)
+    monkeypatch.setattr(setup, "sync_prime_skills", lambda: None)
+
+    setup.run_setup(skip_install=True, skip_agents_md=True, no_interactive=True)
+
+    metadata_path = tmp_path / ".prime" / "lab.json"
+    assert metadata_path.exists()
+    metadata = json.loads(metadata_path.read_text())
+    assert metadata["setup_source"] == "prime lab setup"
+    assert metadata["choices"] == {
+        "agents": ["codex"],
+        "primary_agent": "codex",
+        "use_multiple_agents": False,
+    }

verifiers/scripts/prime_rl.py

@@ -3,7 +3,7 @@
 Wrapper script to run prime-rl rl command from the current working directory.
 
 Usage:
-    uv run prime-rl @ configs/prime-rl/config.toml
+    uv run prime-rl configs/prime-rl/config.toml
 """
 
 import argparse
@@ -147,7 +147,6 @@ def main():
     parser = argparse.ArgumentParser(
         description="Create a tmux session and run prime-rl rl command from a TOML config."
     )
-    parser.add_argument("at", type=str)
     parser.add_argument("config_path", type=str)
     parser.add_argument(
         "--session",
@@ -168,9 +167,6 @@ def main():
     if not tmux_exists():
         raise SystemExit("tmux not found in PATH. Please install tmux.")
 
-    if args.at != "@":
-        raise SystemExit("Usage: prime-rl @ path/to/file.toml")
-
     cwd = Path.cwd()
     prime_rl_dir = cwd / "prime-rl"
     if not prime_rl_dir.exists():