@@ -175,6 +175,20 @@ def _log_non_ok_result(r: EvalResult, status: str, err: str) -> None:
175175 print (f"[WARN] { r .instance_id } status={ status } { detail } error={ err } " , flush = True )
176176
177177
178+ def _maybe_checkpoint (path : Path | None , r : EvalResult , status : str , err : str ) -> None :
179+ if path is None :
180+ return
181+ # Pool-level transient failures (BrokenProcessPool) must NOT be persisted:
182+ # on retry the orchestrator rebuilds the pool and these instances should be
183+ # re-evaluated, not skipped via the resume set.
184+ # EXCEPTION: status=="timeout" is deterministic per-instance (the same input
185+ # would hit the same deadline again) and MUST be checkpointed to prevent an
186+ # infinite retry loop on a pathological repository.
187+ if status != "timeout" and "BrokenProcessPool" in err :
188+ return
189+ append_checkpoint (path , r )
190+
191+
178192def run_eval_set (
179193 instances : list [BenchmarkInstance ],
180194 eval_fn : EvalFn ,
@@ -212,16 +226,7 @@ def _record(r: EvalResult) -> None:
212226 err = str ((r .extra or {}).get ("error" , "" ))
213227 if status not in ("ok" , "empty_query" , "empty_corpus" ) and (status or err ):
214228 _log_non_ok_result (r , status , err )
215- # Pool-level transient failures (BrokenProcessPool) must NOT be
216- # persisted: on retry the orchestrator rebuilds the pool and these
217- # instances should be re-evaluated, not skipped via the resume set.
218- # EXCEPTION: status=="timeout" results — even though the kill switch
219- # surfaces them via BrokenProcessPool, they are deterministic per
220- # instance (the same input would hit the same deadline again) and
221- # MUST be checkpointed to prevent an infinite retry loop on a
222- # pathological repository.
223- if checkpoint_path is not None and (status == "timeout" or "BrokenProcessPool" not in err ):
224- append_checkpoint (checkpoint_path , r )
229+ _maybe_checkpoint (checkpoint_path , r , status , err )
225230
226231 if pending :
227232 if workers <= 1 or len (pending ) <= 1 :
0 commit comments