diff --git a/docs/mint.json b/docs/mint.json index d293c2dbc3..005d0d214d 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -14,7 +14,8 @@ "v1/environments", "v1/evaluation", "v1/harnesses", - "v1/harbor" + "v1/harbor", + "v1/gepa" ] }, { diff --git a/docs/v1/gepa.md b/docs/v1/gepa.md new file mode 100644 index 0000000000..03a3ecdf9b --- /dev/null +++ b/docs/v1/gepa.md @@ -0,0 +1,44 @@ +# GEPA Prompt Optimization + +verifiers offers built in support for [GEPA](https://github.com/gepa-ai/gepa), an algorithm that optimizes a system prompt to maximize the downstream reward for a given taskset: + +```bash +uv run gepa reverse-text-v1 +``` + +`gepa` runs GEPA where a number of rollouts are done before a teacher LLM reflects on the results to propose a better `Task.system_prompt` without any gradient based training. It runs against native v1 tasksets. + +GEPA reuses the same `taskset` / `harness` / `client` / `sampling` config as eval, so the `.toml` config remains very similar: + +```toml +model = "deepseek/deepseek-v4-flash" + +[taskset] +id = "reverse-text-v1" + +[harness] +id = "default" + +[sampling] +temperature = 1.0 +``` + +Validate the config by using `uv run gepa @ config.toml --dry-run`. To run GEPA, use `uv run gepa @ config.toml`. CLI arguments overwrite toml arguments when both are present. + +## Common config values + +- `model` / `-m` — model for the rollouts under optimization (default: `deepseek/deepseek-v4-flash`, same as eval) +- `reflection_model` / `reflection_client` — model/endpoint that proposes new prompts (default: reuse `model` / `client`) +- `num_train` / `num_val` — train tasks for reflection minibatches and held-out val tasks for the pareto frontier (defaults: 100 / 50) +- `max_total_rollouts` — total rollouts the run may spend (default: 500) +- `max_concurrent` / `-c` — caps how many rollouts are in flight at once (default: 128) + +## Output + +Results go under `outputs/----//`, matching `eval`. The best system prompt is printed when the run finishes. + +## Limitations + +**Tasksets** — GEPA optimizes `Task.system_prompt`, so the taskset must provide one. Tasksets that bake instructions into the user `prompt` instead (e.g. `gsm8k-v1`) are not supported out of the box. + +**Harnesses** — any eval harness works. With `APPENDS_SYSTEM_PROMPT`, the optimized prompt is used as a system message but otherwise is folded into the user prompt. diff --git a/docs/v1/overview.md b/docs/v1/overview.md index c870d95d3c..24992169e8 100644 --- a/docs/v1/overview.md +++ b/docs/v1/overview.md @@ -36,5 +36,6 @@ A trace records the message graph, rewards, metrics, errors, etc. When using ver - [Harbor Environments](harbor.md) — How to create Harbor-based environments - [Evaluation](evaluation.md) — How to run said environments - [Harnesses](harnesses.md) — How to build custom harnesses +- [GEPA](gepa.md) — How to optimize system prompts with GEPA For the documentation for legacy environments, go to [the v0 documentation](../v0/overview.md). diff --git a/verifiers/v1/gepa/config.py b/verifiers/v1/gepa/config.py index 5f353bb854..5161c7d0d5 100644 --- a/verifiers/v1/gepa/config.py +++ b/verifiers/v1/gepa/config.py @@ -21,13 +21,14 @@ class GEPAConfig(EnvConfig): """The GEPA run plus its environment. `model` runs the rollouts under optimization; `reflection_model` (defaults to `model`) proposes new system prompts from the reflective - dataset. No default for `model` — a GEPA run can spend a large eval budget, so pick it - explicitly rather than silently optimizing (and spending) against a fallback.""" + dataset. `model` defaults to the same id as `EvalConfig`.""" uuid: str = Field(default_factory=lambda: str(uuid4()), exclude=True) """Auto-generated run id — the leaf of the output dir, so runs never overwrite. Excluded from the saved config so re-running `@ config.toml` lands in a fresh dir.""" - model: str = Field(..., validation_alias=AliasChoices("model", "m")) + model: str = Field( + "deepseek/deepseek-v4-flash", validation_alias=AliasChoices("model", "m") + ) """Model id for rollouts under optimization.""" client: EvalClientConfig = EvalClientConfig() sampling: SamplingConfig = SamplingConfig() @@ -48,14 +49,10 @@ class GEPAConfig(EnvConfig): """Seed for GEPA's optimizer (candidate selection / minibatch sampling). Task shuffling uses a fixed seed, matching eval — so this doesn't change the train/val split.""" - max_metric_calls: int = Field( - 500, validation_alias=AliasChoices("max_metric_calls", "B") - ) + max_total_rollouts: int = Field(500) """Total rollouts GEPA may spend across the whole optimization run.""" reflection_minibatch_size: int = 3 """Train tasks sampled per reflection step.""" - perfect_score: float | None = None - """Skip reflecting on a minibatch that already scores this well.""" reflection_columns: list[str] = Field(default_factory=list) """Extra per-trace fields (from `trace.info`, else `task`) to surface to the teacher LM.""" initial_prompt: str | None = None diff --git a/verifiers/v1/gepa/runner.py b/verifiers/v1/gepa/runner.py index 8a13ce6c87..ff9850d393 100644 --- a/verifiers/v1/gepa/runner.py +++ b/verifiers/v1/gepa/runner.py @@ -101,16 +101,14 @@ async def on_complete(trace: Trace) -> None: valset=[task.data.idx for task in val_tasks], adapter=adapter, reflection_lm=reflection_lm, - max_metric_calls=config.max_metric_calls, + max_metric_calls=config.max_total_rollouts, reflection_minibatch_size=config.reflection_minibatch_size, run_dir=str(run_dir) if run_dir is not None else None, seed=config.seed, display_progress_bar=False, - skip_perfect_score=config.perfect_score is not None, + skip_perfect_score=False, logger=_GEPALog(), ) - if config.perfect_score is not None: - optimize_kwargs["perfect_score"] = config.perfect_score return optimize(**optimize_kwargs) finally: loop.run_until_complete(serving.__aexit__(None, None, None))