|
11 | 11 |
|
12 | 12 | from __future__ import annotations |
13 | 13 |
|
| 14 | +import functools |
14 | 15 | import os |
15 | 16 | import time |
16 | 17 | from pathlib import Path |
@@ -167,23 +168,20 @@ def _bm25_eval( |
167 | 168 | pass |
168 | 169 |
|
169 | 170 |
|
| 171 | +def _pool_eval_bm25(repos_dir_str: str, instance: BenchmarkInstance, params: RunParams) -> EvalResult: |
| 172 | + repos_dir = Path(repos_dir_str) |
| 173 | + worktree_dir = repos_dir / "worktrees" / f"w{os.getpid()}" |
| 174 | + worktree_dir.mkdir(parents=True, exist_ok=True) |
| 175 | + evaluator = UniversalEvaluator() |
| 176 | + encoder = tiktoken.get_encoding("o200k_base") |
| 177 | + return _bm25_eval(instance, params, evaluator, worktree_dir, encoder) |
| 178 | + |
| 179 | + |
170 | 180 | def make_bm25_eval_fn(repos_dir: Path): |
171 | 181 | try: |
172 | 182 | import rank_bm25 as _ # noqa: F401 |
173 | 183 | except ImportError as e: |
174 | 184 | raise RuntimeError( |
175 | 185 | "rank-bm25 not installed; expected to be in requirements-bench.lock. Run: pip install rank-bm25" |
176 | 186 | ) from e |
177 | | - |
178 | | - evaluator = UniversalEvaluator() |
179 | | - worktrees_root = repos_dir / "worktrees" |
180 | | - encoder = tiktoken.get_encoding("o200k_base") |
181 | | - |
182 | | - def eval_fn(instance: BenchmarkInstance, params: RunParams) -> EvalResult: |
183 | | - import os |
184 | | - |
185 | | - worktree_dir = worktrees_root / f"w{os.getpid()}" |
186 | | - worktree_dir.mkdir(parents=True, exist_ok=True) |
187 | | - return _bm25_eval(instance, params, evaluator, worktree_dir, encoder) |
188 | | - |
189 | | - return eval_fn |
| 187 | + return functools.partial(_pool_eval_bm25, str(repos_dir)) |
0 commit comments