fix(bench): per-worker worktree isolation + bake contextbench_verified config

Author: nikolay-e

benchmarks/baselines/aider_baseline.py

@@ -243,11 +243,14 @@ def make_aider_eval_fn(
         raise ValueError(f"aider_mode must be 'fair' or 'oracle', got {aider_mode!r}")
 
     evaluator = UniversalEvaluator()
-    worktree_dir = repos_dir / "worktrees"
-    worktree_dir.mkdir(parents=True, exist_ok=True)
+    worktrees_root = repos_dir / "worktrees"
     aider = _AiderProcess()
 
     def eval_fn(instance: BenchmarkInstance, params: RunParams) -> EvalResult:
+        import os
+
+        worktree_dir = worktrees_root / f"w{os.getpid()}"
+        worktree_dir.mkdir(parents=True, exist_ok=True)
         return _aider_eval(instance, params, evaluator, worktree_dir, aider, request_timeout, aider_mode)
 
     eval_fn.shutdown = aider.shutdown  # type: ignore[attr-defined]

benchmarks/baselines/bm25_baseline.py

@@ -176,11 +176,14 @@ def make_bm25_eval_fn(repos_dir: Path):
         ) from e
 
     evaluator = UniversalEvaluator()
-    worktree_dir = repos_dir / "worktrees"
-    worktree_dir.mkdir(parents=True, exist_ok=True)
+    worktrees_root = repos_dir / "worktrees"
     encoder = tiktoken.get_encoding("o200k_base")
 
     def eval_fn(instance: BenchmarkInstance, params: RunParams) -> EvalResult:
+        import os
+
+        worktree_dir = worktrees_root / f"w{os.getpid()}"
+        worktree_dir.mkdir(parents=True, exist_ok=True)
         return _bm25_eval(instance, params, evaluator, worktree_dir, encoder)
 
     return eval_fn

benchmarks/diffctx_eval_fn.py

@@ -103,7 +103,9 @@ def _arm_diffctx_kill_switch(timeout_s: float):
 def _ensure_worker_state(repos_dir_str: str) -> tuple[Path, UniversalEvaluator]:
     state = _WORKER_STATE.get(repos_dir_str)
     if state is None:
-        worktree_dir = Path(repos_dir_str) / "worktrees" / f"w{os.getpid()}"
+        runner = os.environ.get("RUNNER_NAME", "").replace("-", "_").replace(" ", "_")
+        uid = f"{runner}_{os.getpid()}" if runner else str(os.getpid())
+        worktree_dir = Path(repos_dir_str) / "worktrees" / f"w{uid}"
         worktree_dir.mkdir(parents=True, exist_ok=True)
         evaluator = UniversalEvaluator()
         state = (worktree_dir, evaluator)

scripts/bake_bench_cache.py

@@ -28,6 +28,7 @@
 
 DATASETS: list[tuple[str, str | None, str]] = [
     ("Contextbench/ContextBench", "default", "train"),
+    ("Contextbench/ContextBench", "contextbench_verified", "train"),
     ("princeton-nlp/SWE-bench_Verified", None, "test"),
     ("AmazonScience/SWE-PolyBench_500", None, "test"),
 ]