chore: rename env-rollout to episode in docs and comments (#2095)

mikasenghaas · claude · web-flow · commit cb9c84969186 · 2026-07-21T15:43:51.000-07:00
Standardize terminology on "episode" (the term we use) for the whole
one-task run across agents. Replaces the "env-rollout" / "eval rollout"
wording in docs, skills, code docstrings/comments, tests, and configs.
No behavior change — all edits are prose.

Where a docstring defined the episode in terms of "env-rollout" (episode.py,
env.py, retries.py, agent.py), the wording is rephrased instead of a literal
swap to avoid circular definitions. The generic single-trace "rollout" term
(one task + harness + trace) is left as-is.

Co-authored-by: Claude Opus 4.8 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/configs/code_golf.toml b/configs/code_golf.toml
@@ -1,4 +1,4 @@
-# code-golf-v1 — the taskset ships a recipe env: one env-rollout runs
+# code-golf-v1 — the taskset ships a recipe env: one episode runs
 # `--env.attempts` independent attempts (default 2) and scores them against
 # each other (shortest wins), so one rollout per task is a full comparison.
 #
diff --git a/docs/v1/environments.md b/docs/v1/environments.md
@@ -1,6 +1,6 @@
 # Multi-agent environments
 
-One eval rollout doesn't have to be one agent run. `Env` is abstract, and
+One episode doesn't have to be one agent run. `Env` is abstract, and
 every run gets a concrete subclass: plain tasksets resolve to the bundled
 `SingleAgentEnv` (one `agent` playing the taskset), and a package can export
 its own (via `__all__`, alongside its [`Taskset`](tasksets.md) — the same plugin
diff --git a/docs/v1/overview.md b/docs/v1/overview.md
@@ -22,7 +22,7 @@ An `Agent` is a reusable (harness × model × runtime policy) value with one exe
 
 ## Environment
 
-The control flow between agents: how many run on one task, in what order, judged how across the finished set. The default is the single-agent case; `--env.id` pairs a reusable interaction (best-of-n, a judge) with any taskset. One env-rollout yields flat, self-contained traces — each stamped with its episode and agent — persisted whole (one episode per `traces.jsonl` line).
+The control flow between agents: how many run on one task, in what order, judged how across the finished set. The default is the single-agent case; `--env.id` pairs a reusable interaction (best-of-n, a judge) with any taskset. One episode yields flat, self-contained traces — each stamped with its episode and agent — persisted whole (one episode per `traces.jsonl` line).
 
 ## Toolset
 
diff --git a/docs/v1/tasksets.md b/docs/v1/tasksets.md
@@ -238,6 +238,6 @@ To override the judge model, set `env.taskset.task.judge.model` in your config (
 
 ## Beyond one agent
 
-One eval rollout doesn't have to be one agent run: agents, the control flow between
+One episode doesn't have to be one agent run: agents, the control flow between
 agents, and cross-agent rewards are the environment's job — see
 [Multi-agent environments](environments.md).
diff --git a/environments/code_golf_v1/code_golf_v1/taskset.py b/environments/code_golf_v1/code_golf_v1/taskset.py
@@ -1,6 +1,6 @@
 """code_golf: write a short, fast Python program — the sibling-comparison recipe env.
 
-Each task asks for a tiny program with a known output. The env fans one env-rollout
+Each task asks for a tiny program with a known output. The env fans one episode
 into `--env.attempts` independent attempts by the same "golfer" role and scores them
 against each other. Anything that needs the runtime is measured per attempt, box-live,
 into that attempt's trace; the env's `finalize()` then just compares the recorded
@@ -13,7 +13,7 @@
   - `fastest`       env `finalize()`: of the PASSING attempts, the lowest recorded
                     `latency` wins — a comparison of trace metadata.
 
-So one env-rollout produces, per attempt: did it work, was it the shorter one, was it
+So one episode produces, per attempt: did it work, was it the shorter one, was it
 the quicker one — the relative signals you can only get by comparing siblings. Only
 correct attempts compete: a failed one earns nothing on either comparison, and an
 all-failed group pays nobody.
@@ -69,7 +69,7 @@ async def correct(self, trace: vf.Trace) -> float:
 class CodeGolfEnvConfig(vf.EnvConfig):
     golfer: vf.AgentConfig = vf.AgentConfig()
     attempts: int = Field(2, ge=1)
-    """Independent attempts per env-rollout, scored against each other."""
+    """Independent attempts per episode, scored against each other."""
 
 
 class CodeGolfEnv(vf.Env[CodeGolfEnvConfig]):
diff --git a/environments/proposer_solver_v1/proposer_solver_v1/taskset.py b/environments/proposer_solver_v1/proposer_solver_v1/taskset.py
@@ -78,7 +78,7 @@ class SolveTask(vf.Task[SolveData]):
     @classmethod
     def from_trace(cls, proposer: vf.Trace) -> "SolveTask":
         """trace -> Task: the proposer's JSON contract becomes the solvers' task.
-        Off-contract output raises — the env-rollout fails (retryable) instead of
+        Off-contract output raises — the episode fails (retryable) instead of
         scoring solvers against garbage."""
         reply = proposer.last_reply or ""
         greedy = re.search(r"\{.*\}", reply, re.DOTALL)
diff --git a/skills/create-environments/SKILL.md b/skills/create-environments/SKILL.md
@@ -136,7 +136,7 @@ Runtime config chooses where code executes. Task hooks should use the `vf.Runtim
 - Prefer deterministic verification grounded in the task's actual artifact or answer.
 - Use an LLM judge only when semantic judgment is unavoidable.
 - Metrics are for observability and do not contribute to reward, but are useful. Use them deliberately and appropriately!
-- Judgement that compares the sibling traces of one env-rollout (best-of-n selection, zero-sum payoffs) lives on `Environment.finalize(task, episode)` — attach via `trace.record_reward`/`record_metric`, in program order; no live runtime there.
+- Judgement that compares the sibling traces of one episode (best-of-n selection, zero-sum payoffs) lives on `Environment.finalize(task, episode)` — attach via `trace.record_reward`/`record_metric`, in program order; no live runtime there.
 - Raise ordinary Python exceptions from rollout hooks and scoring. The rollout records them as `TaskError`.
 
 ## Validation and lifecycle
diff --git a/skills/evaluate-environments/SKILL.md b/skills/evaluate-environments/SKILL.md
@@ -160,7 +160,7 @@ outputs/<env>--<model>--<harness>/<uuid>/
 └── eval.log
 ```
 
-Set an exact path with `-o`. `traces.jsonl` is one **episode** per line — an env-rollout's traces plus their shared standing — appended after each episode finishes, so an episode is durable whole or not at all (a torn last line is the whole episode redone on resume).
+Set an exact path with `-o`. `traces.jsonl` is one **episode** per line — the episode's traces plus their shared standing — appended after each episode finishes, so an episode is durable whole or not at all (a torn last line is the whole episode redone on resume).
 
 Resume in place:
 
diff --git a/skills/evaluate-environments/references/REFERENCE.md b/skills/evaluate-environments/references/REFERENCE.md
@@ -45,7 +45,7 @@ Sibling entrypoints reuse the same tree: [`ServeConfig`](#serveconfig--the-env-s
 | `client` | `ClientConfig` | `EvalClientConfig()` | — | The model client (discriminated union — see [Client config](#client-config)). |
 | `sampling` | `SamplingConfig` | `SamplingConfig()` | — | Per-request sampling knobs (see [Sampling config](#sampling-config)). |
 | `num_tasks` | `int \| None` | `None` | `batch_size`, `num_examples`, `num_tasks`, `n` | How many tasks to evaluate (None = all). |
-| `num_rollouts` | `int` | `1` | `group_size`, `rollouts_per_example`, `num_rollouts`, `r` | Independent env-rollouts per task — the trainer's group size. Env-internal fan-out (e.g. best-of-n attempts) is the env's own knob, not `-r`. |
+| `num_rollouts` | `int` | `1` | `group_size`, `rollouts_per_example`, `num_rollouts`, `r` | Independent episodes per task — the trainer's group size. Env-internal fan-out (e.g. best-of-n attempts) is the env's own knob, not `-r`. |
 | `shuffle` | `bool` | `False` | `shuffle`, `s` | Shuffle tasks before taking the first `num_tasks`. |
 | `max_concurrent` | `int \| None` | `128` | `max_concurrent`, `c` | Max rollouts in flight at once. |
 | `verbose` | `bool` | `False` | `verbose`, `v` | Log at debug level instead of info. |
diff --git a/tests/v1/fixtures/duet_v1.py b/tests/v1/fixtures/duet_v1.py
@@ -3,7 +3,7 @@
 The multi-agent fixture for the v1 e2e suite (resolved by id `duet-v1`): an
 `Env` subclass exported alongside its taskset, with two roles ("a", a trainable
 seat on the run's model; "b", pinned untrainable), a `run()` that fans both out on
-the task, and a `finalize()` that records a sibling-dependent signal. One eval rollout
+the task, and a `finalize()` that records a sibling-dependent signal. One episode
 should land one record carrying two role-stamped traces.
 """
 
diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py
@@ -265,8 +265,8 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path):
 @pytest.mark.e2e
 async def test_multi_agent_env(run_v1, tmp_path):
     """An `Env` subclass shipped with its taskset (duet-v1): two roles run the
-    task, `score()` episodes a sibling-dependent metric, and one eval rollout lands one
-    episode carrying two role-stamped traces."""
+    task, `score()` episodes a sibling-dependent metric, and one episode carries
+    two role-stamped traces."""
     import json
 
     traces = await run_v1(
@@ -275,7 +275,7 @@ async def test_multi_agent_env(run_v1, tmp_path):
         output_dir=tmp_path,
         max_turns=2,
     )
-    assert len(traces) == 2  # one env-rollout, one trace per role
+    assert len(traces) == 2  # one episode, one trace per role
     assert sorted(t.agent_name for t in traces) == ["a", "b"]
     (b,) = [t for t in traces if t.agent_name == "b"]
     assert b.trainable is False
@@ -305,7 +305,7 @@ async def test_env_id_best_of_n(run_v1, tmp_path):
         output_dir=tmp_path,
         max_turns=2,
     )
-    assert len(traces) == 2  # one env-rollout, two attempts
+    assert len(traces) == 2  # one episode, two attempts
     assert all(t.agent_name == "agent" and t.ok for t in traces)
     assert any(t.metrics["best"] == 1.0 for t in traces)
     assert all(t.metrics["pass_at_n"] == 1.0 for t in traces)  # echo always passes
diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py
@@ -350,7 +350,7 @@ async def provision(self, task: Task | None = None) -> AsyncIterator[Runtime]:
 
 
 class _EpisodeAgent(Agent):
-    """One role's `Agent` for one env-rollout, built fresh per episode (a cheap
+    """One role's `Agent` for one episode, built fresh each time (a cheap
     bundle of references — expensive resources are env-owned and borrowed, so no
     state spans concurrent episodes): traces get their agent standing the moment
     they're created, finished ones land in `completed` (the episode's traces),
diff --git a/verifiers/v1/cli/dashboard/eval.py b/verifiers/v1/cli/dashboard/eval.py
@@ -291,7 +291,7 @@ def Progress(
 ) -> Group:
     # On resume, `slots` includes the previous session's kept rollouts (as finished slots), so
     # progress, reward, err, and the breakdown cover the whole run, not just this session's.
-    done = [s for s in slots if s.done]  # fully scored env-rollouts
+    done = [s for s in slots if s.done]  # fully scored episodes
     done_traces = [t for s in done for t in s.traces]
     # Score aggregates read the policy's traces: auxiliary roles (a judge's verdict
     # run, a modeled user) are `trainable=False` and carry no rewards, so counting
@@ -300,7 +300,7 @@ def Progress(
     scored = [t for t in done_traces if t.trainable] or done_traces
     total = len(slots)
     # Headline reward = mean over non-errored traces; when any errored, `format_mean` appends
-    # the global avg (errored count as 0) in parens. `err` is the share of env-rollouts that
+    # the global avg (errored count as 0) in parens. `err` is the share of episodes that
     # ended not-ok (a trace errored, or the env's rollout()/score() hook itself failed).
     reward = format_mean(scored, lambda t: t.reward)
     err = (
@@ -477,7 +477,7 @@ def _stage(trace: Trace) -> str:
     """The stage a live (not-yet-done) rollout is in, derived from its trace's timing
     spans — the engine opens and closes each span exactly at the stage transitions, so
     the current stage is the latest span started but not yet ended. A completed trace
-    whose slot isn't done is waiting on its env-rollout's other traces (and the env's
+    whose slot isn't done is waiting on its episode's other traces (and the env's
     `score()`) — that's scoring."""
     if trace.is_completed:
         return "scoring"
@@ -531,7 +531,7 @@ def _brace(i: int, size: int) -> str:
 
 def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
     # (brace, state, left sections, result, time); a slot contributes one row per live
-    # trace (a multi-agent env-rollout shows each role's trace), braced per task.
+    # trace (a multi-agent episode shows each role's trace), braced per task.
     rows: list[tuple[str, str, list[str], str, str]] = []
     for group in groups:
         group_rows: list[tuple[str, list[str], str, str]] = []
@@ -572,7 +572,7 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                         ):  # flag a clipped rollout next to its stop condition
                             stop = f"{stop} (truncated)".strip()
                 elif t.is_completed and (err := t.error) is not None:
-                    # An errored trace whose env-rollout is still running its other
+                    # An errored trace whose episode is still running its other
                     # traces (or `score()`) is already a failure — show it, don't
                     # let it sit as "scoring" until the whole episode lands.
                     state, result, stop = "error", err.type, ""
diff --git a/verifiers/v1/cli/eval/runner.py b/verifiers/v1/cli/eval/runner.py
@@ -1,4 +1,4 @@
-"""The eval runner: fan env-rollouts out with bounded concurrency."""
+"""The eval runner: fan episodes out with bounded concurrency."""
 
 import asyncio
 import contextlib
diff --git a/verifiers/v1/configs/eval.py b/verifiers/v1/configs/eval.py
@@ -33,7 +33,7 @@ class EvalConfig(EnvServerConfig):
             "group_size", "rollouts_per_example", "num_rollouts", "r"
         ),
     )
-    """Independent env-rollouts per task — the trainer's group size."""
+    """Independent episodes per task — the trainer's group size."""
     shuffle: bool = Field(False, validation_alias=AliasChoices("shuffle", "s"))
     """Shuffle tasks before taking the first `num_tasks`."""
     max_concurrent: int | None = Field(
diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py
@@ -1,4 +1,4 @@
-"""Compose a taskset, its agents, and how one task becomes one env-rollout."""
+"""Compose a taskset, its agents, and how one task becomes one episode."""
 
 import asyncio
 import contextlib
@@ -196,7 +196,7 @@ def _as_error(e: Exception) -> Error:
 
 @dataclass
 class RunSlot:
-    """One planned env-rollout, observable while it happens: `traces` collects the
+    """One planned episode, observable while it happens: `traces` collects the
     current attempt (a retry restarts the list), `episode`/`done` land when final."""
 
     task: Task
@@ -218,7 +218,7 @@ def finished(cls, episode: Episode) -> "RunSlot":
 
 
 class Env(ABC, Generic[ConfigT]):
-    """A taskset, the agents that play it, and how one task becomes one env-rollout.
+    """A taskset, the agents that play it, and how one task becomes one episode.
 
     Abstract: every run gets a concrete subclass — `SingleAgentEnv` for every plain
     taskset. A multi-agent env declares each role as an `AgentConfig` field on an
@@ -298,22 +298,22 @@ async def setup(self, agents: Agents) -> None:
 
     @abstractmethod
     async def run(self, task: Task, agents: Agents) -> None:
-        """One env-rollout: how the agents interact on `task`, returning nothing —
+        """One episode: how the agents interact on `task`, returning nothing —
         every finished run joins the episode automatically, stamped with its seat's
         standing. An agent-run failure is data on its trace (this hook decides what
-        it means); an exception raised here is the env-rollout itself failing."""
+        it means); an exception raised here is the episode itself failing."""
 
     async def finalize(self, task: Task, episode: Episode) -> None:
         """Cross-agent judgement — THE programmable judgement surface: plain
         imperative Python over the finished episode (per-trace judgement already
         ran on each trace's own task). `episode.traces` is the flat episode in
         completion order, each trace's `agent_name` stamp naming its agent; attach
         signals via `record_reward`/`record_metric`, in program order. A raise
-        fails the env-rollout (the retryable unit) — validate strictly, never
+        fails the episode (the retryable unit) — validate strictly, never
         record a guess."""
 
     def complete(self, episode: Episode) -> bool:
-        """Whether a finished env-rollout is a valid result — what `--resume` keeps
+        """Whether a finished episode is a valid result — what `--resume` keeps
         vs. redoes (default `episode.ok`). An env whose `run()` tolerates a
         failed participant overrides this, else resume re-runs accepted rollouts."""
         return episode.ok
@@ -340,7 +340,7 @@ def _episode_agents(
         on_trace: Callable[[Trace], None] | None,
         on_discard: Callable[[Trace], None] | None,
     ) -> Agents:
-        """One env-rollout's `Agents` — fresh value objects riding the live serving
+        """One episode's `Agents` — fresh value objects riding the live serving
         resources (nothing shared across concurrent episodes); `setup()` sees them first."""
 
         def make(name: str, spec: AgentConfig) -> Agent:
@@ -391,7 +391,7 @@ async def run_episode(
         on_discard: Callable[[Trace], None] | None = None,
         gate: asyncio.Semaphore | None = None,
     ) -> Episode:
-        """One env-rollout of `task`, minted as the wire atom: `setup()` then `run()`
+        """One episode of `task`, minted as the wire atom: `setup()` then `run()`
         over fresh agents, then `finalize()`; `gate` bounds
         the agent runs, so internal fan-out counts against `--max-concurrent` too.
         Traces join the episode as runs complete — a hook raising mid-way yields the
@@ -440,7 +440,7 @@ async def run_episode(
         return episode
 
     def slots(self, task: Task, n: int = 1) -> list[RunSlot]:
-        """Plan `n` independent env-rollouts of `task` (`-r n`): nothing couples them."""
+        """Plan `n` independent episodes of `task` (`-r n`): nothing couples them."""
         if n < 1:
             raise ValueError("a task needs at least one rollout (n >= 1)")
         return [RunSlot(task) for _ in range(n)]
@@ -452,7 +452,7 @@ async def run_slot(
         semaphore: asyncio.Semaphore | None = None,
         on_complete: Callable[[Episode], Awaitable[None]] | None = None,
     ) -> Episode:
-        """Run one planned env-rollout to its finished episode, with whole-episode
+        """Run one planned episode to completion, with whole-episode
         retries per `--env.retries`; `semaphore` gates the agent RUNS, not the
         episode; `on_complete` (the runners' persistence hook) fires when final."""
 
diff --git a/verifiers/v1/envs/agentic_judge/env.py b/verifiers/v1/envs/agentic_judge/env.py
@@ -152,7 +152,7 @@ async def run(self, task: vf.Task, agents: vf.Agents) -> None:
     async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
         """Record the scraped verdict on the SOLVER's trace. Strict on scale: an
         off-scale score raises (a judge answering `95` must not clamp to full
-        marks), failing the env-rollout rather than scoring the solver wrong."""
+        marks), failing the episode rather than scoring the solver wrong."""
         by_agent = {t.agent_name: t for t in episode.traces}
         solution, verdict = by_agent["solver"], by_agent["judge"]
         data = verdict.info.get("verdict")
diff --git a/verifiers/v1/envs/best_of_n/env.py b/verifiers/v1/envs/best_of_n/env.py
@@ -18,7 +18,7 @@ class BestOfNEnvConfig(vf.EnvConfig):
     # composes with a plain run's `--env.agent.*` flags unchanged.
     agent: vf.AgentConfig = vf.AgentConfig()
     n: int = Field(4, ge=1)
-    """Independent attempts per env-rollout, scored against each other."""
+    """Independent attempts per episode, scored against each other."""
     threshold: float = 1.0
     """A sibling counts as solved when its task reward reaches this (`pass_at_n`)."""
 
diff --git a/verifiers/v1/episode.py b/verifiers/v1/episode.py
@@ -1,4 +1,4 @@
-"""The episode — one env-rollout's traces plus their shared standing, whole."""
+"""The episode — one run's traces plus their shared standing, whole."""
 
 import uuid
 from typing import Generic
@@ -12,7 +12,7 @@
 
 
 class Episode(StrictBaseModel, Generic[DataT, StateT]):
-    """One env-rollout, whole: its identity and standing (`id`, `env`, `errors`)
+    """One run of a task, whole: its identity and standing (`id`, `env`, `errors`)
     next to its flat `traces` — the object `finalize()` receives, the engine
     returns, and the durability envelope: one episode is one `traces.jsonl` line
     and one serve reply, so it persists and arrives whole or not at all — a torn
diff --git a/verifiers/v1/gepa/adapter.py b/verifiers/v1/gepa/adapter.py
diff --git a/verifiers/v1/retries.py b/verifiers/v1/retries.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-# code-golf-v1 — the taskset ships a recipe env: one env-rollout runs`
	`1`	`+# code-golf-v1 — the taskset ships a recipe env: one episode runs`
`2`	`2`	# `--env.attempts` independent attempts (default 2) and scores them against
`3`	`3`	`# each other (shortest wins), so one rollout per task is a full comparison.`
`4`	`4`	`#`