diff --git a/envs/tbench2_env/server/tbench2_env_environment.py b/envs/tbench2_env/server/tbench2_env_environment.py index a232b95c9..f731b1117 100644 --- a/envs/tbench2_env/server/tbench2_env_environment.py +++ b/envs/tbench2_env/server/tbench2_env_environment.py @@ -201,15 +201,36 @@ def _canonical_eval_cmd(workdir: str, timeout_s: float | None = None) -> str: ) -def _parse_canonical_reward(output: str) -> float: +def _parse_canonical_reward(output: str) -> float | None: + """The verdict test.sh's verifier wrote to reward.txt, echoed on the + marker line. None when no verdict can be recovered — no marker line, an + empty value (reward.txt absent: test.sh crashed or was killed before its + verifier wrote one), or a non-numeric value. A missing verdict is a + scoring failure, not a task failure: callers raise so ``evaluate`` + reports an error (observation.error set, reward None) instead of a 0.0 + indistinguishable from tests genuinely failing — RL consumers must be + able to drop such episodes rather than train on a false negative. + """ for line in output.splitlines()[::-1]: if _REWARD_MARKER in line: raw = line.split(_REWARD_MARKER, 1)[1].strip() + if not raw: + return None try: - return float(raw) if raw else 0.0 + return float(raw) except ValueError: - return 0.0 - return 0.0 + return None + return None + + +def _require_canonical_verdict(reward: float | None, output: str) -> float: + """Raise when the canonical harness produced no verdict (see above).""" + if reward is None: + raise RuntimeError( + "canonical harness produced no verdict (reward.txt missing after " + f"tests/test.sh ran); test.sh log tail: {output[-800:]!r}" + ) + return reward def _fallback_eval_cmd(workdir: str) -> str: @@ -624,7 +645,7 @@ def _evaluate_canonical( timeout=timeout_s, ) - reward = _parse_canonical_reward(output) + reward = _require_canonical_verdict(_parse_canonical_reward(output), output) info = {"tests_passed": reward == 1.0, "harness": "tests/test.sh"} return output, reward, info @@ -1028,7 +1049,9 @@ def _evaluate_docker(self) -> tuple[str, float, dict[str, Any]]: _, output = self._exec_in_container( _canonical_eval_cmd(workdir, timeout_s=verifier_timeout_s) ) - reward = _parse_canonical_reward(output) + reward = _require_canonical_verdict( + _parse_canonical_reward(output), output + ) info = {"tests_passed": reward == 1.0, "harness": "tests/test.sh"} else: _, output = self._exec_in_container(_fallback_eval_cmd(workdir)) diff --git a/tests/envs/test_tbench2_env.py b/tests/envs/test_tbench2_env.py index 9640821e8..d9ce71dd2 100644 --- a/tests/envs/test_tbench2_env.py +++ b/tests/envs/test_tbench2_env.py @@ -242,6 +242,27 @@ def test_evaluate_canonical_from_withheld_copy(tmp_path: Path, staged_paths): assert not stage_logs.exists() +def test_evaluate_missing_verdict_raises_not_zero(tmp_path: Path, staged_paths): + """test.sh ran but wrote no reward.txt (crash / kill before the verifier + wrote one): scoring must error out — reward None on the observation — not + return a 0.0 indistinguishable from tests genuinely failing. The staged + assets are still wiped.""" + stage_tests, stage_logs = staged_paths + task = _make_task_dir(tmp_path) + env = Tbench2Environment(withhold_tests=True) + env._task_dir = task + env._workdir = str(tmp_path) + # Marker line present but empty: reward.txt was never written. + env._terminal_toolkit = _RecordingToolkit(output="__TB2_REWARD__:") + env._withhold_verifier_assets(task) + + with pytest.raises(RuntimeError, match="produced no verdict"): + env._evaluate_task() + + assert not stage_tests.exists() + assert not stage_logs.exists() + + def test_evaluate_without_tests_scores_zero(tmp_path: Path, staged_paths): task = tmp_path / "empty-task" task.mkdir() @@ -381,6 +402,22 @@ def test_docker_copy_excludes_verifier_assets(tmp_path: Path): assert len(names) == len(set(names)) # recursive=False: no duplicates +def test_evaluate_docker_missing_verdict_raises_not_zero(tmp_path: Path): + """Docker-mode twin of the local missing-verdict case: an empty reward + marker errors out (with cleanup) instead of scoring 0.0.""" + task = _make_task_dir(tmp_path) + env = Tbench2DockerEnvironment() + container = _FakeContainer(exec_output=b"__TB2_REWARD__:\n") + env._container = container + env._task_dir = task + + with pytest.raises(RuntimeError, match="produced no verdict"): + env._evaluate_docker() + + assert container.events[-1][0] == "exec" + assert "rm -rf /tests /logs/verifier" in container.events[-1][1] + + def test_evaluate_docker_stages_tests_at_verify(tmp_path: Path): task = _make_task_dir(tmp_path) env = Tbench2DockerEnvironment()