fix(tbench2_env): a missing reward.txt is a scoring error, not reward 0.0

envs/tbench2_env/server/tbench2_env_environment.py

@@ -201,15 +201,36 @@ def _canonical_eval_cmd(workdir: str, timeout_s: float | None = None) -> str:
     )
 
 
-def _parse_canonical_reward(output: str) -> float:
+def _parse_canonical_reward(output: str) -> float | None:
+    """The verdict test.sh's verifier wrote to reward.txt, echoed on the
+    marker line. None when no verdict can be recovered — no marker line, an
+    empty value (reward.txt absent: test.sh crashed or was killed before its
+    verifier wrote one), or a non-numeric value. A missing verdict is a
+    scoring failure, not a task failure: callers raise so ``evaluate``
+    reports an error (observation.error set, reward None) instead of a 0.0
+    indistinguishable from tests genuinely failing — RL consumers must be
+    able to drop such episodes rather than train on a false negative.
+    """
     for line in output.splitlines()[::-1]:
         if _REWARD_MARKER in line:
             raw = line.split(_REWARD_MARKER, 1)[1].strip()
+            if not raw:
+                return None
             try:
-                return float(raw) if raw else 0.0
+                return float(raw)
             except ValueError:
-                return 0.0
-    return 0.0
+                return None
+    return None
+
+
+def _require_canonical_verdict(reward: float | None, output: str) -> float:
+    """Raise when the canonical harness produced no verdict (see above)."""
+    if reward is None:
+        raise RuntimeError(
+            "canonical harness produced no verdict (reward.txt missing after "
+            f"tests/test.sh ran); test.sh log tail: {output[-800:]!r}"
+        )
+    return reward
 
 
 def _fallback_eval_cmd(workdir: str) -> str:
@@ -624,7 +645,7 @@ def _evaluate_canonical(
             timeout=timeout_s,
         )
 
-        reward = _parse_canonical_reward(output)
+        reward = _require_canonical_verdict(_parse_canonical_reward(output), output)
         info = {"tests_passed": reward == 1.0, "harness": "tests/test.sh"}
         return output, reward, info
 
@@ -1028,7 +1049,9 @@ def _evaluate_docker(self) -> tuple[str, float, dict[str, Any]]:
                 _, output = self._exec_in_container(
                     _canonical_eval_cmd(workdir, timeout_s=verifier_timeout_s)
                 )
-                reward = _parse_canonical_reward(output)
+                reward = _require_canonical_verdict(
+                    _parse_canonical_reward(output), output
+                )
                 info = {"tests_passed": reward == 1.0, "harness": "tests/test.sh"}
             else:
                 _, output = self._exec_in_container(_fallback_eval_cmd(workdir))

tests/envs/test_tbench2_env.py

@@ -242,6 +242,27 @@ def test_evaluate_canonical_from_withheld_copy(tmp_path: Path, staged_paths):
     assert not stage_logs.exists()
 
 
+def test_evaluate_missing_verdict_raises_not_zero(tmp_path: Path, staged_paths):
+    """test.sh ran but wrote no reward.txt (crash / kill before the verifier
+    wrote one): scoring must error out — reward None on the observation — not
+    return a 0.0 indistinguishable from tests genuinely failing. The staged
+    assets are still wiped."""
+    stage_tests, stage_logs = staged_paths
+    task = _make_task_dir(tmp_path)
+    env = Tbench2Environment(withhold_tests=True)
+    env._task_dir = task
+    env._workdir = str(tmp_path)
+    # Marker line present but empty: reward.txt was never written.
+    env._terminal_toolkit = _RecordingToolkit(output="__TB2_REWARD__:")
+    env._withhold_verifier_assets(task)
+
+    with pytest.raises(RuntimeError, match="produced no verdict"):
+        env._evaluate_task()
+
+    assert not stage_tests.exists()
+    assert not stage_logs.exists()
+
+
 def test_evaluate_without_tests_scores_zero(tmp_path: Path, staged_paths):
     task = tmp_path / "empty-task"
     task.mkdir()
@@ -381,6 +402,22 @@ def test_docker_copy_excludes_verifier_assets(tmp_path: Path):
     assert len(names) == len(set(names))  # recursive=False: no duplicates
 
 
+def test_evaluate_docker_missing_verdict_raises_not_zero(tmp_path: Path):
+    """Docker-mode twin of the local missing-verdict case: an empty reward
+    marker errors out (with cleanup) instead of scoring 0.0."""
+    task = _make_task_dir(tmp_path)
+    env = Tbench2DockerEnvironment()
+    container = _FakeContainer(exec_output=b"__TB2_REWARD__:\n")
+    env._container = container
+    env._task_dir = task
+
+    with pytest.raises(RuntimeError, match="produced no verdict"):
+        env._evaluate_docker()
+
+    assert container.events[-1][0] == "exec"
+    assert "rm -rf /tests /logs/verifier" in container.events[-1][1]
+
+
 def test_evaluate_docker_stages_tests_at_verify(tmp_path: Path):
     task = _make_task_dir(tmp_path)
     env = Tbench2DockerEnvironment()