Skip to content

Commit 01ee4fa

Browse files
committed
fix(bench): extract _maybe_checkpoint to reduce run_eval_set complexity
1 parent 5e7bd62 commit 01ee4fa

1 file changed

Lines changed: 15 additions & 10 deletions

File tree

benchmarks/adapters/runner.py

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,20 @@ def _log_non_ok_result(r: EvalResult, status: str, err: str) -> None:
175175
print(f"[WARN] {r.instance_id} status={status}{detail} error={err}", flush=True)
176176

177177

178+
def _maybe_checkpoint(path: Path | None, r: EvalResult, status: str, err: str) -> None:
179+
if path is None:
180+
return
181+
# Pool-level transient failures (BrokenProcessPool) must NOT be persisted:
182+
# on retry the orchestrator rebuilds the pool and these instances should be
183+
# re-evaluated, not skipped via the resume set.
184+
# EXCEPTION: status=="timeout" is deterministic per-instance (the same input
185+
# would hit the same deadline again) and MUST be checkpointed to prevent an
186+
# infinite retry loop on a pathological repository.
187+
if status != "timeout" and "BrokenProcessPool" in err:
188+
return
189+
append_checkpoint(path, r)
190+
191+
178192
def run_eval_set(
179193
instances: list[BenchmarkInstance],
180194
eval_fn: EvalFn,
@@ -212,16 +226,7 @@ def _record(r: EvalResult) -> None:
212226
err = str((r.extra or {}).get("error", ""))
213227
if status not in ("ok", "empty_query", "empty_corpus") and (status or err):
214228
_log_non_ok_result(r, status, err)
215-
# Pool-level transient failures (BrokenProcessPool) must NOT be
216-
# persisted: on retry the orchestrator rebuilds the pool and these
217-
# instances should be re-evaluated, not skipped via the resume set.
218-
# EXCEPTION: status=="timeout" results — even though the kill switch
219-
# surfaces them via BrokenProcessPool, they are deterministic per
220-
# instance (the same input would hit the same deadline again) and
221-
# MUST be checkpointed to prevent an infinite retry loop on a
222-
# pathological repository.
223-
if checkpoint_path is not None and (status == "timeout" or "BrokenProcessPool" not in err):
224-
append_checkpoint(checkpoint_path, r)
229+
_maybe_checkpoint(checkpoint_path, r, status, err)
225230

226231
if pending:
227232
if workers <= 1 or len(pending) <= 1:

0 commit comments

Comments
 (0)