fix(bench): make aider eval fn picklable for pebble workers

Author: nikolay-e

QA.md

@@ -188,6 +188,41 @@
   matching feature list (`lang-core` or `lang-extra`) WITHOUT `dep:` prefix,
   add `#[cfg]`-gated insert in `tree_sitter_strategy.rs::LANGUAGE_CACHE`
 
+## Windows CI Compatibility
+
+- `fcntl` is Unix-only; `msvcrt.locking` is the Windows equivalent. For code
+  that runs on CI (not on the bench server), guard Unix-only imports:
+
+  ```python
+  try:
+      import fcntl as _fcntl
+  except ImportError:
+      _fcntl = None  # type: ignore[assignment]
+  ```
+
+- Bench jobs (common.py, diffctx_eval_fn.py) import `fcntl` only for the
+  `_cache_lock` function — benchmark code never runs on Windows, but the
+  test suite imports these modules. The `try/except` pattern keeps tests
+  runnable on Windows while bench logic remains Linux-only.
+
+## Bench Sweep Smoke QA
+
+- **Always read GH Actions logs in full before diagnosing failures** — summary
+  counts (n_ok/n_total) do not distinguish between bugs and expected infra failures.
+  Two common false-positive patterns:
+  - `oom_kill` (SIGKILL -9) on large repos like `astropy/astropy` — expected on GH
+    runners with 7GB RAM; not a bug in the algorithm.
+  - `clone_fail` / `clone_timeout` — transient network or GH rate-limit; retry or
+    skip, not a code regression.
+- **Picklable eval functions** — eval_fn passed to pebble `ProcessPool.schedule()`
+  must be picklable. Local closures (`def eval_fn(...)` inside a factory function)
+  are NOT picklable. Use `functools.partial` + module-level function.
+  Attaching `.shutdown` as a lambda is also not picklable — use a module-level
+  `_noop_shutdown` function.
+- `make_aider_eval_fn` was fixed to use `functools.partial(_pool_eval_aider, ...)`.
+  The `_AiderProcess` is now created lazily per-worker-process via a module-level
+  `_AIDER_PROC` global (no inter-process sharing needed).
+
 ## actionlint context restrictions
 
 - `${{ env.X }}` is BANNED inside `env:` blocks at any level (workflow, job,
@@ -242,8 +277,6 @@ this QA pass (touching benchmark code = regression risk on v1 results):
   `# noqa: C901` for ruff (orchestration with timeout/exception/checkpoint dispatch
   does not factor cleanly without behavior change)
 - `benchmarks/adapters/evaluator.py:49` (32→15)
-- `scripts/bake_bench_cache.py:72` (16→15)
-
 Tactic from earlier passes (already in this file): extract `_collect_result`
 helpers and `_print_*_header`/`_print_*_dump` blocks. Pair with v2 evaluation
 re-run so any drift is caught.

benchmarks/baselines/aider_baseline.py

@@ -5,13 +5,14 @@
 hard-pins ~95 deps including litellm, numpy==1.26.4, fastapi — those would
 break the main treemapper env if installed in-process).
 
-Spawn-once, reuse-many: a single helper process is kept alive across all
-instances in a run, so we pay aider's import cost (~1-2s) once per
-benchmark file, not 1500 times.
+Spawn-once, reuse-many: one helper process per worker process is kept alive
+across all instances assigned to that worker, so we pay aider's import
+cost (~1-2s) once per worker, not once per instance.
 """
 
 from __future__ import annotations
 
+import functools
 import json
 import os
 import shutil
@@ -234,24 +235,36 @@ def _aider_eval(
             pass
 
 
+_AIDER_PROC: _AiderProcess | None = None
+
+
+def _noop_shutdown() -> None:
+    pass
+
+
+def _pool_eval_aider(
+    repos_dir_str: str,
+    request_timeout: float,
+    aider_mode: str,
+    instance: BenchmarkInstance,
+    params: RunParams,
+) -> EvalResult:
+    global _AIDER_PROC
+    if _AIDER_PROC is None:
+        _AIDER_PROC = _AiderProcess()
+    evaluator = UniversalEvaluator()
+    worktree_dir = Path(repos_dir_str) / "worktrees" / f"w{os.getpid()}"
+    worktree_dir.mkdir(parents=True, exist_ok=True)
+    return _aider_eval(instance, params, evaluator, worktree_dir, _AIDER_PROC, request_timeout, aider_mode)
+
+
 def make_aider_eval_fn(
     repos_dir: Path,
     request_timeout: float = 300.0,
     aider_mode: str = "fair",
 ):
     if aider_mode not in {"fair", "oracle"}:
         raise ValueError(f"aider_mode must be 'fair' or 'oracle', got {aider_mode!r}")
-
-    evaluator = UniversalEvaluator()
-    worktrees_root = repos_dir / "worktrees"
-    aider = _AiderProcess()
-
-    def eval_fn(instance: BenchmarkInstance, params: RunParams) -> EvalResult:
-        import os
-
-        worktree_dir = worktrees_root / f"w{os.getpid()}"
-        worktree_dir.mkdir(parents=True, exist_ok=True)
-        return _aider_eval(instance, params, evaluator, worktree_dir, aider, request_timeout, aider_mode)
-
-    eval_fn.shutdown = aider.shutdown  # type: ignore[attr-defined]
-    return eval_fn
+    fn = functools.partial(_pool_eval_aider, str(repos_dir), request_timeout, aider_mode)
+    fn.shutdown = _noop_shutdown  # type: ignore[attr-defined]
+    return fn