Fix v1 task example id normalization

Author: willccbb

docs/evaluation.md

@@ -136,7 +136,7 @@ env.set_concurrency(256)
 | `--api-client-type` | — | `openai_chat_completions` | Client type: `openai_completions`, `openai_chat_completions`, `openai_chat_completions_token`, `openai_responses`, `renderer`, `anthropic_messages`, or `nemorl_chat_completions` |
 | `--endpoints-path` | `-e` | `./configs/endpoints.toml` | Path to TOML endpoints registry |
 | `--header` | — | — | Extra HTTP header (`Name: Value`), repeatable |
-| `--header-from-state` | — | `X-Session-ID: example_id` | Per-request header whose value is read from rollout state (`Name: state_key`), repeatable |
+| `--header-from-state` | — | framework session id | Per-request header whose value is read from rollout state (`Name: state_key`), repeatable |
 
 The `renderer` client type requires the optional renderer package. Install it with `uv add "verifiers[renderers]"` before running evals with `--api-client-type renderer`.
 
@@ -178,7 +178,7 @@ headers = { "X-Custom-Header" = "value" }
 
 In `[[eval]]` TOML configs you can set extra headers as `headers = { ... }` and/or as a list `header = ["Name: Value", ...]` (same form as repeated `--header`). Merge order is: registry row, then the `headers` table, then each `header` / `--header` line, with later entries overriding the same name.
 
-For per-request headers that need to vary per rollout (e.g. sticky DP-aware routing keyed off `example_id` or `trajectory_id`), use `headers_from_state = { "X-Name" = "state_key" }` and/or `header_from_state = ["X-Name: state_key", ...]` (same form as repeated `--header-from-state`). The value for each request is resolved at send time as `state[state_key]`. If unset, `X-Session-ID` defaults to `example_id`.
+For per-request headers that need to vary per rollout, use `headers_from_state = { "X-Name" = "state_key" }` and/or `header_from_state = ["X-Name: state_key", ...]` (same form as repeated `--header-from-state`). The value for each request is resolved at send time as `state[state_key]`. If unset, Verifiers supplies a framework-managed `X-Session-ID`.
 
 To define equivalent replicas, add multiple `[[endpoint]]` entries with the same `endpoint_id`.
 

docs/reference.md

@@ -126,7 +126,7 @@ Selects which `Client` implementation to use. Set via `ClientConfig.client_type`
 
 ```python
 class State(dict):
-    INPUT_FIELDS = ["prompt", "answer", "info", "example_id"]
+    INPUT_FIELDS = ["prompt", "answer", "info"]
 ```
 
 A `dict` subclass that tracks rollout information. Accessing keys in `INPUT_FIELDS` automatically forwards to the nested `input` object.
@@ -162,7 +162,6 @@ A `dict` subclass that tracks rollout information. Accessing keys in `INPUT_FIEL
 ```python
 class RolloutInput(TypedDict):
     prompt: Messages        # Required
-    example_id: int         # Framework-managed
     answer: str             # Optional
     info: Info              # Optional
 ```
@@ -172,7 +171,6 @@ class RolloutInput(TypedDict):
 ```python
 class RolloutOutput(dict):
     # Required fields
-    example_id: int  # Framework-managed
     prompt: Messages | None
     completion: Messages | None
     reward: float

docs/training.md

@@ -42,17 +42,20 @@ configs/
 │   ├── qwen-3-5.toml
 │   ├── qwen-3-5-moe.toml
 │   ├── nemotron-3.toml
-│   └── llama-3.toml
+│   ├── llama-3.toml
+│   └── gpt-oss.toml
 ├── rl/
 │   ├── qwen-3-5.toml
 │   ├── qwen-3-5-moe.toml
 │   ├── nemotron-3.toml
-│   └── llama-3.toml
+│   ├── llama-3.toml
+│   └── gpt-oss.toml
 └── gepa/
     ├── qwen-3-5.toml
     ├── qwen-3-5-moe.toml
     ├── nemotron-3.toml
-    └── llama-3.toml
+    ├── llama-3.toml
+    └── gpt-oss.toml
 ```
 
 Example configuration file for the `primeintellect/reverse-text` environment with `Qwen/Qwen3.5-4B`:

tests/test_gepa_cli.py

@@ -189,6 +189,14 @@ def test_load_gepa_toml_config_requires_env_table(tmp_path: Path):
         load_gepa_toml_config(config_path)
 
 
+def test_repo_gepa_example_configs_are_valid():
+    config_paths = sorted(Path("configs/gepa").glob("*.toml"))
+    assert config_paths
+    for config_path in config_paths:
+        loaded = load_gepa_toml_config(config_path)
+        assert loaded["envs"], f"{config_path} should contain at least one [[env]]"
+
+
 def test_resolve_gepa_config_args_supports_plain_env_id():
     args = argparse.Namespace(env_id_or_config="primeintellect/wordle")
 

tests/test_save_utils.py

@@ -258,6 +258,13 @@ def test_states_to_outputs(self, make_state):
         assert result[0].get("foo") == "bar"  # custom field from make_state fixture
         assert result[0]["reward"] == 1.0
 
+    def test_states_to_outputs_requires_example_id(self, make_state):
+        state = make_state()
+        del state["example_id"]
+
+        with pytest.raises(KeyError):
+            states_to_outputs([state], state_columns=[])
+
     def test_states_to_outputs_completion_keeps_messages(self, make_state):
         states = [
             make_state(
@@ -647,6 +654,22 @@ def test_builder_uses_custom_threshold(self):
         # 1 of 4 correct at threshold=0.7: pass^1 = C(1,1)/C(4,1) = 0.25
         assert metadata["pass_all_k"]["1"] == pytest.approx(0.25)
 
+    def test_builder_requires_example_id(self):
+        builder = GenerateOutputsBuilder(
+            env_id="test-env",
+            env_args={},
+            model="test-model",
+            client=ClientConfig(api_base_url="http://localhost:8000/v1"),
+            num_examples=1,
+            rollouts_per_example=1,
+            state_columns=[],
+            sampling_args={},
+            results_path=Path("/tmp/test-results"),
+        )
+
+        with pytest.raises(KeyError):
+            builder.add_outputs([{"reward": 1.0, "metrics": {}}])
+
 
 class TestMetricProtocol:
     def test_all_metrics_satisfy_protocol(self):

tests/test_v1_taskset_utils.py

@@ -0,0 +1,46 @@
+import json
+
+from datasets import Dataset
+
+from verifiers.v1.utils.taskset_utils import dataset_from_result
+
+
+def task_payload(row: dict) -> dict:
+    return json.loads(row["info"]["task"])
+
+
+def test_dataset_from_result_assigns_example_id_to_iterable_records():
+    dataset = dataset_from_result(
+        [
+            {"question": "Reverse abc.", "answer": "cba"},
+            {"question": "Reverse xyz.", "answer": "zyx"},
+        ],
+        "ReverseTextTaskset",
+    )
+
+    rows = list(dataset)
+    payloads = [task_payload(row) for row in rows]
+
+    assert [row["example_id"] for row in rows] == [0, 1]
+    assert [payload["example_id"] for payload in payloads] == [0, 1]
+    assert all(len(payload["task_id"]) == 32 for payload in payloads)
+    assert {payload["task_id"] for payload in payloads}.isdisjoint({"0", "1"})
+
+
+def test_dataset_from_result_overwrites_existing_example_id_column():
+    raw_dataset = Dataset.from_list(
+        [
+            {"question": "Reverse abc.", "answer": "cba", "example_id": None},
+            {"question": "Reverse xyz.", "answer": "zyx", "example_id": 99},
+        ]
+    )
+
+    dataset = dataset_from_result(raw_dataset, "ReverseTextTaskset")
+
+    rows = list(dataset)
+    payloads = [task_payload(row) for row in rows]
+
+    assert [row["example_id"] for row in rows] == [0, 1]
+    assert [payload["example_id"] for payload in payloads] == [0, 1]
+    assert all(len(payload["task_id"]) == 32 for payload in payloads)
+    assert {payload["task_id"] for payload in payloads}.isdisjoint({"0", "1", "99"})

verifiers/utils/metric_utils.py

@@ -162,10 +162,12 @@ def __init__(self, rollouts_per_example: int, threshold: float = 0.5) -> None:
         self.reset()
 
     def add_output(self, output: RolloutOutput) -> None:
+        example_id = output["example_id"]
+        if example_id is None:
+            raise ValueError("output['example_id'] is required.")
         if not self._k_values:
             return
 
-        example_id = output.get("example_id", 0)
         self._example_counts[example_id] += 1
         if output.get("reward", 0.0) >= self.threshold:
             self._example_correct[example_id] += 1

verifiers/utils/save_utils.py

@@ -218,8 +218,12 @@ def state_to_output(
     else:
         raise TypeError("state['timing'] must be a RolloutTiming or mapping.")
 
+    example_id = state["example_id"]
+    if example_id is None:
+        raise ValueError("state['example_id'] is required.")
+
     output = RolloutOutput(
-        example_id=state.get("example_id", 0),
+        example_id=example_id,
         prompt=state.get("prompt"),
         completion=state.get("completion"),
         answer=state.get("answer", ""),
@@ -671,9 +675,16 @@ def build_metadata(self) -> GenerateMetadata:
     def build_outputs(self, sort_by_example_id: bool = False) -> list[RolloutOutput]:
         """Return (sorted) accumulated outputs"""
         if sort_by_example_id:
-            return sorted(self.outputs, key=lambda o: o.get("example_id", 0))
+            return sorted(self.outputs, key=self.output_example_id)
         return self.outputs
 
+    @staticmethod
+    def output_example_id(output: RolloutOutput) -> int:
+        example_id = output["example_id"]
+        if example_id is None:
+            raise ValueError("output['example_id'] is required.")
+        return example_id
+
     def build(self, sort_by_example_id: bool = False) -> GenerateOutputs:
         """Build GenerateOutputs from accumulated outputs."""
         return GenerateOutputs(

verifiers/v1/utils/taskset_utils.py

@@ -38,10 +38,8 @@ def prepare_task(task: Task, taskset_id: str) -> Task:
         raise TypeError("v1 task loaders must return Task objects.")
     prepared = Task(cast(JsonData, dict(task)))
     prepared["taskset_id"] = taskset_id
-    if "task_id" in prepared:
+    if prepared.get("task_id") is not None:
         prepared["task_id"] = str(prepared["task_id"])
-    elif "example_id" in prepared:
-        prepared["task_id"] = str(prepared["example_id"])
     else:
         prepared["task_id"] = uuid.uuid4().hex
     return prepared.freeze()
@@ -51,13 +49,13 @@ def dataset_record_from_task(
     task: Task,
     taskset_id: str,
     index: int,
-    source: JsonData | None = None,
+    record: JsonData | None = None,
 ) -> JsonData:
     data = Task(cast(JsonData, dict(task)))
-    data.setdefault("example_id", source.get("example_id") if source else index)
+    data["example_id"] = index
     normalized = prepare_task(data, taskset_id)
     task_payload = dict(normalized)
-    dataset_record = deepcopy(dict(source or {}))
+    dataset_record = deepcopy(dict(record or {}))
     dataset_record["prompt"] = task_payload["prompt"]
     dataset_record["example_id"] = task_payload["example_id"]
     info = dataset_record.get("info")
@@ -82,9 +80,10 @@ def dataset_from_result(result: Tasks, taskset_id: str) -> Dataset:
     if isinstance(result, Dataset):
         records: list[JsonData] = []
         for index, record in enumerate(result):
-            source = cast(JsonData, dict(record))
-            task = task_from_dataset_record(source, taskset_id)
-            records.append(dataset_record_from_task(task, taskset_id, index, source))
+            row = cast(JsonData, dict(record))
+            row["example_id"] = index
+            task = task_from_dataset_record(row, taskset_id)
+            records.append(dataset_record_from_task(task, taskset_id, index, row))
         return Dataset.from_list(records)
     tasks = tasks_from_result(result, taskset_id)
     return Dataset.from_list(dataset_records_from_tasks(tasks, taskset_id))