From fb61777f02e44300d94e9f39e682472596d490e5 Mon Sep 17 00:00:00 2001 From: Noah Ziems Date: Wed, 15 Jul 2026 12:25:42 -0700 Subject: [PATCH 1/3] feat(v1): share taskset system_prompt overrides with GEPA and eval Add Judge-style system_prompt / system_prompt_file on TasksetConfig so GEPA best prompts can be reused in eval and training without editing tasksets. Co-authored-by: Cursor --- docs/v1/evaluation.md | 3 +++ docs/v1/gepa.md | 11 ++++++++-- .../color_codeword_v1/taskset.py | 3 ++- .../reverse_text_v1/taskset.py | 3 ++- .../references/REFERENCE.md | 2 ++ verifiers/v1/__init__.py | 3 ++- verifiers/v1/gepa/config.py | 5 +---- verifiers/v1/gepa/dataset.py | 17 ++++++++-------- verifiers/v1/gepa/runner.py | 10 ++++++++-- verifiers/v1/taskset.py | 20 ++++++++++++++++++- verifiers/v1/tasksets/lean/taskset.py | 6 +++--- verifiers/v1/tasksets/textarena/taskset.py | 3 ++- 12 files changed, 61 insertions(+), 25 deletions(-) diff --git a/docs/v1/evaluation.md b/docs/v1/evaluation.md index 3dd0cb0dc..5c4b124f9 100644 --- a/docs/v1/evaluation.md +++ b/docs/v1/evaluation.md @@ -33,6 +33,9 @@ The output from evaluations are written into `outputs/--------//`, matching `eval`. The best system prompt is printed when the run finishes. +Results go under `outputs/----//`, matching `eval`. The best system prompt is written to `best_system_prompt.txt` and printed when the run finishes. + +Reuse it in eval or training via the same taskset knobs: + +```bash +uv run eval reverse-text-v1 \ + --taskset.system-prompt-file outputs//best_system_prompt.txt +``` ## Limitations **Tasksets** — GEPA optimizes `Task.system_prompt`, so the taskset must provide one. Tasksets that bake instructions into the user `prompt` instead (e.g. `gsm8k-v1`) are not supported out of the box. -**Harnesses** — any eval harness works. With `APPENDS_SYSTEM_PROMPT`, the optimized prompt is used as a system message but otherwise is folded into the user prompt. +**Harnesses** — any eval harness works. With `APPENDS_SYSTEM_PROMPT`, the optimized prompt is used as a system message; otherwise it is folded into the user prompt. diff --git a/environments/color_codeword_v1/color_codeword_v1/taskset.py b/environments/color_codeword_v1/color_codeword_v1/taskset.py index b44f90548..2d273d995 100644 --- a/environments/color_codeword_v1/color_codeword_v1/taskset.py +++ b/environments/color_codeword_v1/color_codeword_v1/taskset.py @@ -131,6 +131,7 @@ def load(self) -> Iterator[ColorCodewordTask]: for color in colors } length = c.images_per_turn * MAX_TURNS + system_prompt = vf.resolve_system_prompt(c) or SYSTEM_PROMPT for idx in itertools.count(): sequence = [rng.choice(colors) for _ in range(length)] answer = "".join(COLOR_MAP[col] for col in sequence) @@ -148,7 +149,7 @@ def load(self) -> Iterator[ColorCodewordTask]: ColorCodewordTaskData( idx=idx, prompt=[vf.UserMessage(content=parts)], - system_prompt=SYSTEM_PROMPT, + system_prompt=system_prompt, answer=answer, info={ "colors_per_turn": colors_per_turn, diff --git a/environments/reverse_text_v1/reverse_text_v1/taskset.py b/environments/reverse_text_v1/reverse_text_v1/taskset.py index 21d20ed80..934f7a8ef 100644 --- a/environments/reverse_text_v1/reverse_text_v1/taskset.py +++ b/environments/reverse_text_v1/reverse_text_v1/taskset.py @@ -46,12 +46,13 @@ def load(self) -> list[ReverseTextTask]: from datasets import load_dataset rows = load_dataset(self.config.dataset_name, split=self.config.dataset_split) + system_prompt = vf.resolve_system_prompt(self.config) or SYSTEM return [ ReverseTextTask( ReverseTextData( idx=i, prompt=row["prompt"], - system_prompt=SYSTEM, + system_prompt=system_prompt, answer=row["prompt"][::-1], ), self.config.task, diff --git a/skills/evaluate-environments/references/REFERENCE.md b/skills/evaluate-environments/references/REFERENCE.md index cd9a11800..9bbfccb74 100644 --- a/skills/evaluate-environments/references/REFERENCE.md +++ b/skills/evaluate-environments/references/REFERENCE.md @@ -216,6 +216,8 @@ fields become typed dotted flags such as `--taskset.split test`. |---|---|---|---| | `id` | `ID` | `""` | Local package or Hub `org/name[@version]`; selects the taskset and its config type. Set via `--taskset.id`. | | `task` | `TaskConfig` | `TaskConfig()` | Task-facing config passed to every constructed task. `SerializeAsAny` preserves a narrowed subclass. Set through `--taskset.task.*`. | +| `system_prompt` | `str \| None` | `None` | Override `TaskData.system_prompt` for tasksets that honor it in `load()`. Mutually exclusive with `system_prompt_file`. | +| `system_prompt_file` | `Path \| None` | `None` | File override for `system_prompt` (read as UTF-8 text), same mutual-exclusion pattern as `JudgeConfig.prompt` / `prompt_file`. | `.name` → the package name (id with org / version stripped). diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 7a693248f..1796daaf4 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -90,7 +90,7 @@ TaskTimeout, WireTaskData, ) -from verifiers.v1.taskset import Taskset, TasksetConfig +from verifiers.v1.taskset import Taskset, TasksetConfig, resolve_system_prompt from verifiers.v1.mcp import ( Toolset, SharedToolsetConfig, @@ -196,6 +196,7 @@ "Taskset", "TaskConfig", "TasksetConfig", + "resolve_system_prompt", "BaseConfig", "Harness", "HarnessConfig", diff --git a/verifiers/v1/gepa/config.py b/verifiers/v1/gepa/config.py index 5161c7d0d..f1611c22b 100644 --- a/verifiers/v1/gepa/config.py +++ b/verifiers/v1/gepa/config.py @@ -55,9 +55,6 @@ class GEPAConfig(EnvConfig): """Train tasks sampled per reflection step.""" reflection_columns: list[str] = Field(default_factory=list) """Extra per-trace fields (from `trace.info`, else `task`) to surface to the teacher LM.""" - initial_prompt: str | None = None - """Seed system prompt. None = the first loaded task's `Task.system_prompt`, if any task - sets one (see `resolve_gepa_seed_prompt`).""" max_concurrent: int | None = Field( 128, validation_alias=AliasChoices("max_concurrent", "c") @@ -67,7 +64,7 @@ class GEPAConfig(EnvConfig): None, validation_alias=AliasChoices("output_dir", "o") ) """Where to write results (config.toml + the streamed traces.jsonl, alongside GEPA's own - candidates.json / run_log.json). None = a fresh per-run dir under + candidates.json / run_log.json / best_system_prompt.txt). None = a fresh per-run dir under `outputs/----/` (via `output_path`).""" save_results: bool = True verbose: bool = Field(False, validation_alias=AliasChoices("verbose", "v")) diff --git a/verifiers/v1/gepa/dataset.py b/verifiers/v1/gepa/dataset.py index a7178c118..0e2e74317 100644 --- a/verifiers/v1/gepa/dataset.py +++ b/verifiers/v1/gepa/dataset.py @@ -37,23 +37,22 @@ def split_tasks( return tasks[:num_train], tasks[num_train : num_train + num_val] -def resolve_gepa_seed_prompt(tasks: list[Task], initial_prompt: str | None) -> str: - """The system prompt GEPA starts optimizing from: `initial_prompt` if given, else the first - task that sets `system_prompt`. Some tasksets (e.g. `gsm8k-v1`) bake instructions into - `prompt` rather than `system_prompt` and can't be optimized this way — pass `--initial-prompt` - to seed one explicitly. +def resolve_gepa_seed_prompt(tasks: list[Task]) -> str: + """The system prompt GEPA starts optimizing from: the first task that sets + `system_prompt`. Override via `--taskset.system-prompt` / `--taskset.system-prompt-file` + on tasksets that honor those fields in `load()`. Some tasksets (e.g. `gsm8k-v1`) bake + instructions into `prompt` rather than `system_prompt` and can't be optimized this way. How the resolved prompt reaches the model at rollout time — a real system message vs. folded into the user prompt — is the harness's call: `Harness.resolve_prompt` owns that policy and warns when it folds, so it isn't re-policed here.""" - if initial_prompt is not None: - return initial_prompt for task in tasks: if task.data.system_prompt is not None: return task.data.system_prompt raise ValueError( "no task in this taskset sets Task.system_prompt — some tasksets bake instructions " "directly into `prompt` instead (e.g. gsm8k-v1) and can't be optimized this way. Pass " - "--initial-prompt to seed one explicitly, or pick a taskset whose load() sets " - "system_prompt on its task data (e.g. reverse-text-v1, lean, textarena)." + "--taskset.system-prompt or --taskset.system-prompt-file (on a taskset whose load() " + "honors them), or pick a taskset that sets system_prompt by default (e.g. " + "reverse-text-v1, lean, textarena)." ) diff --git a/verifiers/v1/gepa/runner.py b/verifiers/v1/gepa/runner.py index ff9850d39..5401bceb4 100644 --- a/verifiers/v1/gepa/runner.py +++ b/verifiers/v1/gepa/runner.py @@ -46,7 +46,7 @@ def run_gepa(env: Environment, config: GEPAConfig) -> GEPAResult: ) # GEPA scores n=1 per task; groups need >=2 # Seed from the tasks GEPA actually evaluates (train ∪ val), not the full pre-split pool — # a taskset with per-task system prompts could otherwise seed from a task in neither split. - seed_prompt = resolve_gepa_seed_prompt(selected_tasks, config.initial_prompt) + seed_prompt = resolve_gepa_seed_prompt(selected_tasks) tasks_by_idx = {task.data.idx: task for task in selected_tasks} run_dir = output_path(config) if config.save_results else None @@ -109,7 +109,13 @@ async def on_complete(trace: Trace) -> None: skip_perfect_score=False, logger=_GEPALog(), ) - return optimize(**optimize_kwargs) + result = optimize(**optimize_kwargs) + if run_dir is not None: + best = result.best_candidate.get("system_prompt", "") + path = run_dir / "best_system_prompt.txt" + path.write_text(best, encoding="utf-8") + logger.info("best system prompt: %s", path) + return result finally: loop.run_until_complete(serving.__aexit__(None, None, None)) finally: diff --git a/verifiers/v1/taskset.py b/verifiers/v1/taskset.py index 0cf9c2abb..0a463a53a 100644 --- a/verifiers/v1/taskset.py +++ b/verifiers/v1/taskset.py @@ -30,9 +30,10 @@ def load(self) -> Iterable[MyTask]: import itertools import logging from collections.abc import Iterable +from pathlib import Path from typing import TYPE_CHECKING, ClassVar, Generic -from pydantic import SerializeAsAny +from pydantic import SerializeAsAny, model_validator from pydantic_config import BaseConfig from typing_extensions import TypeVar @@ -52,6 +53,16 @@ class TasksetConfig(BaseConfig): """Local package or Hub `org/name[@version]`, set with `--taskset.id`.""" task: SerializeAsAny[TaskConfig] = TaskConfig() """Config passed to each task, under `--taskset.task.*`.""" + system_prompt: str | None = None + """Override `TaskData.system_prompt` for tasksets that honor it in `load()`.""" + system_prompt_file: Path | None = None + """File override for `system_prompt`, mutually exclusive with `system_prompt`.""" + + @model_validator(mode="after") + def check_system_prompt_source(self) -> TasksetConfig: + if self.system_prompt is not None and self.system_prompt_file is not None: + raise ValueError("set `system_prompt` or `system_prompt_file`, not both") + return self @property def name(self) -> str: @@ -61,6 +72,13 @@ def name(self) -> str: TasksetConfigT = TypeVar("TasksetConfigT", bound=TasksetConfig, default=TasksetConfig) +def resolve_system_prompt(config: TasksetConfig) -> str | None: + """Inline `system_prompt`, else contents of `system_prompt_file`, else None.""" + if config.system_prompt_file is not None: + return config.system_prompt_file.read_text(encoding="utf-8") + return config.system_prompt + + class Taskset(Generic[TaskT, TasksetConfigT]): INFINITE: ClassVar[bool] = False """Whether `load` yields tasks forever (a procedural generator). Infinity is inherent diff --git a/verifiers/v1/tasksets/lean/taskset.py b/verifiers/v1/tasksets/lean/taskset.py index 4031330d2..ead8e73a9 100644 --- a/verifiers/v1/tasksets/lean/taskset.py +++ b/verifiers/v1/tasksets/lean/taskset.py @@ -18,7 +18,7 @@ from verifiers.v1.runtimes import Runtime from verifiers.v1.state import State from verifiers.v1.task import Task, TaskConfig, TaskData, TaskResources -from verifiers.v1.taskset import Taskset, TasksetConfig +from verifiers.v1.taskset import Taskset, TasksetConfig, resolve_system_prompt from verifiers.v1.tasksets.lean.scoring import ( build_starter_file, expected_protected_signature, @@ -59,7 +59,6 @@ class LeanTaskConfig(TaskConfig): class LeanConfig(TasksetConfig): dataset: LeanDatasetConfig docker_image: str = DEFAULT_DOCKER_IMAGE - system_prompt: str = DEFAULT_SYSTEM_PROMPT task: LeanTaskConfig = LeanTaskConfig() @@ -169,6 +168,7 @@ def load(self) -> Iterator[LeanTask]: ) resources = TaskResources(cpu=4, memory=4, disk=10) + system_prompt = resolve_system_prompt(config) or DEFAULT_SYSTEM_PROMPT for index, row in enumerate(raw): # An empty statement would reduce the signature guard to `:= by`. formal_statement = row[ds.statement_column] @@ -185,7 +185,7 @@ def load(self) -> Iterator[LeanTask]: idx=index, name=str(name) if name else f"task_{index:05d}", prompt=self._build_prompt(formal_statement, header), - system_prompt=config.system_prompt, + system_prompt=system_prompt, image=config.docker_image, workdir=config.task.lean_project_path, resources=resources, diff --git a/verifiers/v1/tasksets/textarena/taskset.py b/verifiers/v1/tasksets/textarena/taskset.py index 80575cbc3..e3d39d99b 100644 --- a/verifiers/v1/tasksets/textarena/taskset.py +++ b/verifiers/v1/tasksets/textarena/taskset.py @@ -135,13 +135,14 @@ def observation(seed: int) -> str: # seed-invariant; otherwise each task must expose its seeded board. first = observation(0) seed_specific = observation(1) != first + system_prompt = vf.resolve_system_prompt(self.config) or SYSTEM_PROMPT for i in itertools.count(): yield TextArenaTask( TextArenaData( idx=i, name=f"{self.config.game}#{i}", prompt=observation(i) if seed_specific else first, - system_prompt=SYSTEM_PROMPT, + system_prompt=system_prompt, info={"game": self.config.game, "seed": i}, ), self.config.task, From 0c6b6ce4f84b2bcd51466d44ceafaa1a7aac3bcf Mon Sep 17 00:00:00 2001 From: Noah Ziems Date: Wed, 15 Jul 2026 12:32:40 -0700 Subject: [PATCH 2/3] docs(v1): restore GEPA harness limitation wording Co-authored-by: Cursor --- docs/v1/gepa.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/v1/gepa.md b/docs/v1/gepa.md index cffa95112..17990e09b 100644 --- a/docs/v1/gepa.md +++ b/docs/v1/gepa.md @@ -48,4 +48,4 @@ uv run eval reverse-text-v1 \ **Tasksets** — GEPA optimizes `Task.system_prompt`, so the taskset must provide one. Tasksets that bake instructions into the user `prompt` instead (e.g. `gsm8k-v1`) are not supported out of the box. -**Harnesses** — any eval harness works. With `APPENDS_SYSTEM_PROMPT`, the optimized prompt is used as a system message; otherwise it is folded into the user prompt. +**Harnesses** — any eval harness works. With `APPENDS_SYSTEM_PROMPT`, the optimized prompt is used as a system message but otherwise is folded into the user prompt. From 902c959425588f07e62d2d297d64e794af9a7d10 Mon Sep 17 00:00:00 2001 From: Noah Ziems Date: Wed, 15 Jul 2026 15:12:47 -0700 Subject: [PATCH 3/3] wip(v1): apply config system prompts after load Framework overlay via iter_tasks; GEPA targets the config layer; default harness composes harness then task. Co-authored-by: Cursor --- docs/v1/evaluation.md | 5 +- docs/v1/gepa.md | 4 +- .../color_codeword_v1/taskset.py | 3 +- .../reverse_text_v1/taskset.py | 3 +- .../references/REFERENCE.md | 4 +- verifiers/v1/cli/gepa.py | 5 +- verifiers/v1/gepa/__init__.py | 5 +- verifiers/v1/gepa/adapter.py | 29 +++++----- verifiers/v1/gepa/config.py | 5 +- verifiers/v1/gepa/dataset.py | 31 +++++----- verifiers/v1/gepa/runner.py | 11 ++-- verifiers/v1/harness.py | 13 ++++- verifiers/v1/harnesses/default/harness.py | 19 ++++--- verifiers/v1/serve/server.py | 10 ++-- verifiers/v1/taskset.py | 56 ++++++++++++++----- verifiers/v1/tasksets/lean/taskset.py | 5 +- verifiers/v1/tasksets/textarena/taskset.py | 3 +- 17 files changed, 130 insertions(+), 81 deletions(-) diff --git a/docs/v1/evaluation.md b/docs/v1/evaluation.md index 5c4b124f9..19f47b1a9 100644 --- a/docs/v1/evaluation.md +++ b/docs/v1/evaluation.md @@ -33,9 +33,8 @@ The output from evaluations are written into `outputs/---- Iterator[ColorCodewordTask]: for color in colors } length = c.images_per_turn * MAX_TURNS - system_prompt = vf.resolve_system_prompt(c) or SYSTEM_PROMPT for idx in itertools.count(): sequence = [rng.choice(colors) for _ in range(length)] answer = "".join(COLOR_MAP[col] for col in sequence) @@ -149,7 +148,7 @@ def load(self) -> Iterator[ColorCodewordTask]: ColorCodewordTaskData( idx=idx, prompt=[vf.UserMessage(content=parts)], - system_prompt=system_prompt, + system_prompt=SYSTEM_PROMPT, answer=answer, info={ "colors_per_turn": colors_per_turn, diff --git a/environments/reverse_text_v1/reverse_text_v1/taskset.py b/environments/reverse_text_v1/reverse_text_v1/taskset.py index 934f7a8ef..21d20ed80 100644 --- a/environments/reverse_text_v1/reverse_text_v1/taskset.py +++ b/environments/reverse_text_v1/reverse_text_v1/taskset.py @@ -46,13 +46,12 @@ def load(self) -> list[ReverseTextTask]: from datasets import load_dataset rows = load_dataset(self.config.dataset_name, split=self.config.dataset_split) - system_prompt = vf.resolve_system_prompt(self.config) or SYSTEM return [ ReverseTextTask( ReverseTextData( idx=i, prompt=row["prompt"], - system_prompt=system_prompt, + system_prompt=SYSTEM, answer=row["prompt"][::-1], ), self.config.task, diff --git a/skills/evaluate-environments/references/REFERENCE.md b/skills/evaluate-environments/references/REFERENCE.md index 9bbfccb74..771463d06 100644 --- a/skills/evaluate-environments/references/REFERENCE.md +++ b/skills/evaluate-environments/references/REFERENCE.md @@ -216,7 +216,7 @@ fields become typed dotted flags such as `--taskset.split test`. |---|---|---|---| | `id` | `ID` | `""` | Local package or Hub `org/name[@version]`; selects the taskset and its config type. Set via `--taskset.id`. | | `task` | `TaskConfig` | `TaskConfig()` | Task-facing config passed to every constructed task. `SerializeAsAny` preserves a narrowed subclass. Set through `--taskset.task.*`. | -| `system_prompt` | `str \| None` | `None` | Override `TaskData.system_prompt` for tasksets that honor it in `load()`. Mutually exclusive with `system_prompt_file`. | +| `system_prompt` | `str \| None` | `None` | Config-layer system prompt: replaces bake-in `TaskData.system_prompt` after `load()` (GEPA optimizes this layer). Mutually exclusive with `system_prompt_file`. | | `system_prompt_file` | `Path \| None` | `None` | File override for `system_prompt` (read as UTF-8 text), same mutual-exclusion pattern as `JudgeConfig.prompt` / `prompt_file`. | `.name` → the package name (id with org / version stripped). @@ -432,7 +432,7 @@ value in the run's `TimeoutConfig` wins; otherwise the corresponding row value i | `name` | `str \| None` | `None` | Optional human-readable label used in logs and dashboards. | | `description` | `str \| None` | `None` | Optional human-readable description. | | `prompt` | `str \| Messages \| None` | — | Initial user input. A string is one user prompt; `Messages` seeds a full initial conversation and requires a harness with `SUPPORTS_MESSAGE_PROMPT`; `None` lets the user simulator open via `respond("")`. | -| `system_prompt` | `str \| None` | `None` | Optional system prompt. Harnesses with `APPENDS_SYSTEM_PROMPT` emit a real system message; otherwise a string prompt is prefixed with a warning. A separate system prompt cannot be folded into `Messages` or `None`. | +| `system_prompt` | `str \| None` | `None` | Bake-in task-side system prompt (per-task or taskset default). Config `taskset.system_prompt` / `_file` replaces it after `load()`. Harnesses with `APPENDS_SYSTEM_PROMPT` emit a real system message (harness framing first via `compose_system_prompt` when set); otherwise a string prompt is prefixed with a warning. A separate system prompt cannot be folded into `Messages` or `None`. | | `image` | `str \| None` | `None` | Required container/sandbox image for this row. It replaces the base runtime image; subprocess is refused when set. | | `workdir` | `str \| None` | `None` | Working directory for harness execution and task hooks. Applied when the runtime supports it and its config remains at the default. | | `timeout` | `TaskTimeout` | `TaskTimeout()` | Per-stage timeout requests described above. | diff --git a/verifiers/v1/cli/gepa.py b/verifiers/v1/cli/gepa.py index cafff2106..43657aefe 100644 --- a/verifiers/v1/cli/gepa.py +++ b/verifiers/v1/cli/gepa.py @@ -1,7 +1,8 @@ """The GEPA entrypoint: `uv run gepa [] --model [options]`. -Registered as the `gepa` console script. Optimizes a v1 taskset's `Task.system_prompt` via -GEPA (Genetic-Pareto): alternating rollouts with a teacher LM reflecting on results — see +Registered as the `gepa` console script. Optimizes a v1 taskset's config-layer system +prompt (`--taskset.system-prompt` / `--taskset.system-prompt-file`) via GEPA +(Genetic-Pareto): alternating rollouts with a teacher LM reflecting on results — see `verifiers.v1.gepa`. CLI resolution mirrors `eval`/`serve` (`verifiers.v1.cli.resolve`): a leading bare token is the taskset id, the taskset/harness subconfigs are narrowed from their `id`s so `--taskset.*` / `--harness.*` stay typed and `-h` renders them, `@ file.toml` loads, diff --git a/verifiers/v1/gepa/__init__.py b/verifiers/v1/gepa/__init__.py index 6d1b48fb8..bb0d04b49 100644 --- a/verifiers/v1/gepa/__init__.py +++ b/verifiers/v1/gepa/__init__.py @@ -1,8 +1,9 @@ """GEPA (Genetic-Pareto) system-prompt optimization for native v1 environments. `run_gepa(env, config)` drives the third-party `gepa` optimizer against a v1 `Environment` -via `GEPAAdapter`, seeding from and improving `Task.system_prompt`. The `gepa` console -script (`verifiers.v1.cli.gepa`) is a thin entrypoint over this package. +via `GEPAAdapter`, seeding from and improving the taskset config-layer system prompt +(`--taskset.system-prompt` / `--taskset.system-prompt-file`). The `gepa` console script +(`verifiers.v1.cli.gepa`) is a thin entrypoint over this package. """ from verifiers.v1.gepa.adapter import GEPAAdapter diff --git a/verifiers/v1/gepa/adapter.py b/verifiers/v1/gepa/adapter.py index f98b627ae..fa5492616 100644 --- a/verifiers/v1/gepa/adapter.py +++ b/verifiers/v1/gepa/adapter.py @@ -26,12 +26,14 @@ @dataclass class GEPAAdapter: - """Bridges GEPA's optimization loop with a native v1 `Environment`. `tasks` covers only the - trainset + valset tasks GEPA was given (not the whole taskset), keyed by `task.data.idx` — - GEPA's `batch` is a list of those idxs, and injecting the candidate rebuilds each `Task` - around a `data` row carrying the new `system_prompt`. `loop` is the runner's persistent event - loop (which holds `env.serving()` open); `evaluate` drives its rollouts on it with - `run_until_complete`.""" + """Bridges GEPA's optimization loop with a native v1 `Environment`. + + `tasks` are bake-in rows (pre config-overlay), keyed by `task.data.idx`. Each + candidate is applied as a config-layer override via + `Taskset.apply_system_prompt(..., override=...)` — the same path as + `--taskset.system-prompt`. `loop` holds `env.serving()` open; `evaluate` drives + rollouts with `run_until_complete`. + """ env: Environment ctx: ModelContext @@ -53,9 +55,10 @@ def evaluate( candidate: Candidate, capture_traces: bool = False, ) -> EvaluationBatch[Trace, Trace]: - """Run `candidate`'s system prompt on the tasks named by `batch` (`Task.idx` values) - and score them. Called synchronously by GEPA on the main thread; each batch's rollouts - run on the runner's persistent loop via `run_until_complete`.""" + """Run `candidate`'s config-layer system prompt on the tasks named by `batch` + (`Task.idx` values) and score them. Called synchronously by GEPA on the main + thread; each batch's rollouts run on the runner's persistent loop via + `run_until_complete`.""" system_prompt = candidate.get("system_prompt", "") traces = self.loop.run_until_complete(self._run_batch(batch, system_prompt)) scores = [trace.reward for trace in traces] @@ -66,13 +69,11 @@ def evaluate( ) async def _run_batch(self, batch: list[int], system_prompt: str) -> list[Trace]: - # Inject the candidate by rebuilding each Task around a data row with the new - # system_prompt (TaskData is frozen; behavior/config carry over unchanged). tasks = [ - type(t)( - t.data.model_copy(update={"system_prompt": system_prompt}), t.config + self.env.taskset.apply_system_prompt( + self.tasks[idx], override=system_prompt ) - for t in (self.tasks[idx] for idx in batch) + for idx in batch ] episodes = [self.env.episode(task, self.ctx, n=1) for task in tasks] results = await asyncio.gather( diff --git a/verifiers/v1/gepa/config.py b/verifiers/v1/gepa/config.py index f1611c22b..a4c090484 100644 --- a/verifiers/v1/gepa/config.py +++ b/verifiers/v1/gepa/config.py @@ -1,7 +1,8 @@ """The `GEPAConfig`: the single config object the `gepa` CLI parses. -GEPA optimizes one taskset's `Task.system_prompt` by alternating rollouts (`evaluate`) with a -teacher LM reflecting on the reflective dataset (`make_reflective_dataset`) — see +GEPA optimizes the taskset config-layer system prompt (`taskset.system_prompt` / +`system_prompt_file`) by alternating rollouts (`evaluate`) with a teacher LM reflecting +on the reflective dataset (`make_reflective_dataset`) — see `verifiers.v1.gepa.adapter.GEPAAdapter`. This inherits `EnvConfig`'s fields (`taskset`, `harness`, `max_turns`, token limits, timeouts) as top-level flags, the same way `EvalConfig` does, and adds the optimization loop's own knobs (model, reflection model, train/val split, diff --git a/verifiers/v1/gepa/dataset.py b/verifiers/v1/gepa/dataset.py index 0e2e74317..1fc0b4b76 100644 --- a/verifiers/v1/gepa/dataset.py +++ b/verifiers/v1/gepa/dataset.py @@ -8,6 +8,7 @@ from verifiers.v1.decorators import discover_decorated from verifiers.v1.task import Task +from verifiers.v1.taskset import TasksetConfig, resolve_system_prompt def reject_group_reward_tasksets(tasks: list[Task]) -> None: @@ -37,22 +38,26 @@ def split_tasks( return tasks[:num_train], tasks[num_train : num_train + num_val] -def resolve_gepa_seed_prompt(tasks: list[Task]) -> str: - """The system prompt GEPA starts optimizing from: the first task that sets - `system_prompt`. Override via `--taskset.system-prompt` / `--taskset.system-prompt-file` - on tasksets that honor those fields in `load()`. Some tasksets (e.g. `gsm8k-v1`) bake - instructions into `prompt` rather than `system_prompt` and can't be optimized this way. +def resolve_gepa_seed_prompt(config: TasksetConfig, tasks: list[Task]) -> str: + """The config-layer system prompt GEPA starts from. - How the resolved prompt reaches the model at rollout time — a real system message vs. folded - into the user prompt — is the harness's call: `Harness.resolve_prompt` owns that policy and - warns when it folds, so it isn't re-policed here.""" + Prefer `--taskset.system-prompt` / `--taskset.system-prompt-file` when set; otherwise + bootstrap from the first bake-in `TaskData.system_prompt` on `tasks` (expected to be + pre-overlay / `select(apply_config=False)`). Tasksets that only put instructions in + `prompt` (e.g. `gsm8k-v1`) need an explicit config seed. + + How the prompt reaches the model — a real system message vs. folded into the user + prompt — is the harness's call (`Harness.resolve_prompt` / `compose_system_prompt`). + """ + if (override := resolve_system_prompt(config)) is not None: + return override for task in tasks: if task.data.system_prompt is not None: return task.data.system_prompt raise ValueError( - "no task in this taskset sets Task.system_prompt — some tasksets bake instructions " - "directly into `prompt` instead (e.g. gsm8k-v1) and can't be optimized this way. Pass " - "--taskset.system-prompt or --taskset.system-prompt-file (on a taskset whose load() " - "honors them), or pick a taskset that sets system_prompt by default (e.g. " - "reverse-text-v1, lean, textarena)." + "no config system prompt and no task sets TaskData.system_prompt — some " + "tasksets bake instructions into `prompt` instead (e.g. gsm8k-v1). Pass " + "--taskset.system-prompt or --taskset.system-prompt-file as the GEPA seed, " + "or pick a taskset that sets a bake-in system_prompt (e.g. reverse-text-v1, " + "lean, textarena)." ) diff --git a/verifiers/v1/gepa/runner.py b/verifiers/v1/gepa/runner.py index 5401bceb4..9952a38ab 100644 --- a/verifiers/v1/gepa/runner.py +++ b/verifiers/v1/gepa/runner.py @@ -38,15 +38,18 @@ def log(self, message: str) -> None: def run_gepa(env: Environment, config: GEPAConfig) -> GEPAResult: logger.info("gepa config:\n%s", config.model_dump_json(indent=2)) - all_tasks = env.taskset.select(config.num_train + config.num_val, config.shuffle) + all_tasks = env.taskset.select( + config.num_train + config.num_val, + config.shuffle, + apply_config=False, + ) train_tasks, val_tasks = split_tasks(all_tasks, config.num_train, config.num_val) selected_tasks = [*train_tasks, *val_tasks] reject_group_reward_tasksets( selected_tasks ) # GEPA scores n=1 per task; groups need >=2 - # Seed from the tasks GEPA actually evaluates (train ∪ val), not the full pre-split pool — - # a taskset with per-task system prompts could otherwise seed from a task in neither split. - seed_prompt = resolve_gepa_seed_prompt(selected_tasks) + # Config-layer seed (explicit override, else bake-in from train ∪ val tasks). + seed_prompt = resolve_gepa_seed_prompt(env.taskset.config, selected_tasks) tasks_by_idx = {task.data.idx: task for task in selected_tasks} run_dir = output_path(config) if config.save_results else None diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index 15766ad9a..99e36566d 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -50,7 +50,9 @@ def resolved_env(self) -> dict[str, str]: class Harness(ABC, Generic[ConfigT]): APPENDS_SYSTEM_PROMPT: ClassVar[bool] = False - """Emit `TaskData.system_prompt` separately instead of folding it into the user prompt.""" + """Emit the task-side system prompt as a separate system message instead of folding + it into the user prompt. (Harness-owned framing is composed separately via + `compose_system_prompt` — harness first, then task.)""" SUPPORTS_MCP: ClassVar[bool] = False SUPPORTS_USER_SIM: ClassVar[bool] = False SUPPORTS_MESSAGE_PROMPT: ClassVar[bool] = False @@ -58,6 +60,15 @@ class Harness(ABC, Generic[ConfigT]): def __init__(self, config: ConfigT) -> None: self.config = config + def system_prompt(self) -> str | None: + """Harness-owned framing (tools/protocol). None if none.""" + return None + + def compose_system_prompt(self, task_system: str | None) -> str | None: + """Harness framing first, then the task-side system prompt.""" + parts = [p for p in (self.system_prompt(), task_system) if p] + return "\n\n".join(parts) or None + def resolve_prompt( self, task: TaskData ) -> tuple[str | None, str | Messages | None]: diff --git a/verifiers/v1/harnesses/default/harness.py b/verifiers/v1/harnesses/default/harness.py index 50de87edf..d489916f3 100644 --- a/verifiers/v1/harnesses/default/harness.py +++ b/verifiers/v1/harnesses/default/harness.py @@ -42,6 +42,14 @@ class DefaultHarness(Harness[DefaultHarnessConfig]): SUPPORTS_USER_SIM = True SUPPORTS_MESSAGE_PROMPT = True + def system_prompt(self) -> str | None: + fragments = [BASH_SYSTEM_PROMPT] + if self.config.edit: + fragments.append(EDIT_SYSTEM_PROMPT) + if self.config.search: + fragments.append(SEARCH_PROMPT) + return " ".join(fragments) + async def setup(self, runtime: Runtime) -> None: await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env) @@ -54,15 +62,8 @@ async def launch( secret: str, mcp_urls: dict[str, str], ) -> ProgramResult: - system_prompt, prompt = self.resolve_prompt(trace.task.data) - fragments = [BASH_SYSTEM_PROMPT] - if self.config.edit: - fragments.append(EDIT_SYSTEM_PROMPT) - if self.config.search: - fragments.append(SEARCH_PROMPT) - system_prompt = "\n\n".join( - p for p in (" ".join(fragments), system_prompt) if p - ) + task_system, prompt = self.resolve_prompt(trace.task.data) + system_prompt = self.compose_system_prompt(task_system) env = {**self.config.resolved_env} args = [ f"--base-url={endpoint}", diff --git a/verifiers/v1/serve/server.py b/verifiers/v1/serve/server.py index 1396d9858..80a87f5cb 100644 --- a/verifiers/v1/serve/server.py +++ b/verifiers/v1/serve/server.py @@ -40,7 +40,7 @@ def __init__( # A finite taskset is materialized up front (its count is served via `info`); an # infinite one is pulled off its generator on demand (see `_task`), so # `num_tasks=None` on the wire ⟺ the taskset is infinite. - self._task_iter = iter(self.env.taskset.load()) + self._task_iter = iter(self.env.taskset.iter_tasks()) self._tasks: list = [] self.num_tasks: int | None = None if not type(self.env.taskset).INFINITE: @@ -88,10 +88,10 @@ def run_server(cls, address_queue=None, **kwargs) -> None: def _task(self, idx: int): """The task at `idx`; an infinite taskset is generated (and cached) up to `idx` on demand. Generation must be deterministic — every pool worker runs its own - `load()`, so idx-addressing relies on all of them producing the same sequence. - Lazy generation is capped at `MAX_LAZY_TASKS`: an idx that far ahead is a - runaway driver, and generating (and caching) toward it would hang the worker - and exhaust memory instead of failing the one request.""" + `iter_tasks()`, so idx-addressing relies on all of them producing the same + sequence. Lazy generation is capped at `MAX_LAZY_TASKS`: an idx that far ahead + is a runaway driver, and generating (and caching) toward it would hang the + worker and exhaust memory instead of failing the one request.""" while len(self._tasks) <= idx: if idx >= MAX_LAZY_TASKS: raise IndexError( diff --git a/verifiers/v1/taskset.py b/verifiers/v1/taskset.py index 0a463a53a..75b1475e1 100644 --- a/verifiers/v1/taskset.py +++ b/verifiers/v1/taskset.py @@ -9,8 +9,8 @@ def load(self) -> Iterable[MyTask]: `load` may also be a generator — yielding each task as it's built, possibly forever (a procedural taskset; declare `INFINITE = True`). Runs materialize the tasks they need -through `select`, which pulls only that many off a generator; the env server instead -pulls `load` on demand, task by task. +through `select` / `iter_tasks`, which pull off `load` and apply the config-layer +`system_prompt` / `system_prompt_file` override; the env server uses `iter_tasks` too. Load-time knobs (dataset, split, seed) live on the taskset config; the task-facing knobs under its `task` subtree (`TasksetConfig.task`, a `TaskConfig`, everything under @@ -54,7 +54,8 @@ class TasksetConfig(BaseConfig): task: SerializeAsAny[TaskConfig] = TaskConfig() """Config passed to each task, under `--taskset.task.*`.""" system_prompt: str | None = None - """Override `TaskData.system_prompt` for tasksets that honor it in `load()`.""" + """Config-layer system prompt: replaces baked-in `TaskData.system_prompt` after `load()` + (GEPA optimizes this layer; pass a GEPA `best_system_prompt.txt` via `system_prompt_file`).""" system_prompt_file: Path | None = None """File override for `system_prompt`, mutually exclusive with `system_prompt`.""" @@ -94,15 +95,44 @@ def __init__(self, config: TasksetConfigT) -> None: def load(self) -> Iterable[TaskT]: raise NotImplementedError + def apply_system_prompt(self, task: TaskT, override: str | None = None) -> TaskT: + """Config-layer replace of `TaskData.system_prompt`. + + When `override` is omitted, reads `resolve_system_prompt(self.config)`. GEPA + passes each candidate as `override` so rollouts share the same path as + `--taskset.system-prompt` / `--taskset.system-prompt-file` without mutating + `TasksetConfig`. + """ + prompt = resolve_system_prompt(self.config) if override is None else override + if prompt is None or task.data.system_prompt == prompt: + return task + return type(task)( + task.data.model_copy(update={"system_prompt": prompt}), + task.config, + ) + + def iter_tasks(self) -> Iterable[TaskT]: + """`load()` with the config-layer system prompt overlay applied.""" + for task in self.load(): + yield self.apply_system_prompt(task) + def select( - self, num_tasks: int | None = None, shuffle: bool = False + self, + num_tasks: int | None = None, + shuffle: bool = False, + *, + apply_config: bool = True, ) -> list[TaskT]: - """Materialize the tasks a run needs: the first `num_tasks` off `load` (all of - them when `None`), pulled lazily — a generator `load` only builds what the run - takes. `shuffle` samples the subset from the whole taskset instead (the shared - fixed-seed shuffle, `verifiers.v1.utils.sampling`), which materializes everything - first; on an `INFINITE` taskset it's a no-op (warned) — the first `num_tasks` - generated tasks are already an arbitrary sample.""" + """Materialize the tasks a run needs: the first `num_tasks` off `iter_tasks` + (or `load` when `apply_config=False`), all of them when `None`, pulled lazily — + a generator `load` only builds what the run takes. `shuffle` samples the subset + from the whole taskset instead (the shared fixed-seed shuffle, + `verifiers.v1.utils.sampling`), which materializes everything first; on an + `INFINITE` taskset it's a no-op (warned) — the first `num_tasks` generated + tasks are already an arbitrary sample. GEPA passes `apply_config=False` so the + adapter can apply each candidate via `apply_system_prompt` onto bake-in prompts. + """ + source: Iterable[TaskT] = self.iter_tasks() if apply_config else self.load() if type(self).INFINITE: if num_tasks is None: raise ValueError( @@ -115,10 +145,10 @@ def select( "taking the first %d generated tasks", num_tasks, ) - return list(itertools.islice(self.load(), num_tasks)) + return list(itertools.islice(source, num_tasks)) if shuffle: - return sample(self.load(), shuffle=True, limit=num_tasks) - return list(itertools.islice(self.load(), num_tasks)) + return sample(source, shuffle=True, limit=num_tasks) + return list(itertools.islice(source, num_tasks)) def server_config(self, server_cls: type) -> BaseConfig: """The config a `tools` entry is built with, resolved off `self.config` (the diff --git a/verifiers/v1/tasksets/lean/taskset.py b/verifiers/v1/tasksets/lean/taskset.py index ead8e73a9..dc637d03d 100644 --- a/verifiers/v1/tasksets/lean/taskset.py +++ b/verifiers/v1/tasksets/lean/taskset.py @@ -18,7 +18,7 @@ from verifiers.v1.runtimes import Runtime from verifiers.v1.state import State from verifiers.v1.task import Task, TaskConfig, TaskData, TaskResources -from verifiers.v1.taskset import Taskset, TasksetConfig, resolve_system_prompt +from verifiers.v1.taskset import Taskset, TasksetConfig from verifiers.v1.tasksets.lean.scoring import ( build_starter_file, expected_protected_signature, @@ -168,7 +168,6 @@ def load(self) -> Iterator[LeanTask]: ) resources = TaskResources(cpu=4, memory=4, disk=10) - system_prompt = resolve_system_prompt(config) or DEFAULT_SYSTEM_PROMPT for index, row in enumerate(raw): # An empty statement would reduce the signature guard to `:= by`. formal_statement = row[ds.statement_column] @@ -185,7 +184,7 @@ def load(self) -> Iterator[LeanTask]: idx=index, name=str(name) if name else f"task_{index:05d}", prompt=self._build_prompt(formal_statement, header), - system_prompt=system_prompt, + system_prompt=DEFAULT_SYSTEM_PROMPT, image=config.docker_image, workdir=config.task.lean_project_path, resources=resources, diff --git a/verifiers/v1/tasksets/textarena/taskset.py b/verifiers/v1/tasksets/textarena/taskset.py index e3d39d99b..80575cbc3 100644 --- a/verifiers/v1/tasksets/textarena/taskset.py +++ b/verifiers/v1/tasksets/textarena/taskset.py @@ -135,14 +135,13 @@ def observation(seed: int) -> str: # seed-invariant; otherwise each task must expose its seeded board. first = observation(0) seed_specific = observation(1) != first - system_prompt = vf.resolve_system_prompt(self.config) or SYSTEM_PROMPT for i in itertools.count(): yield TextArenaTask( TextArenaData( idx=i, name=f"{self.config.game}#{i}", prompt=observation(i) if seed_specific else first, - system_prompt=system_prompt, + system_prompt=SYSTEM_PROMPT, info={"game": self.config.game, "seed": i}, ), self.config.task,