|
90 | 90 | { |
91 | 91 | "id": "4.1", "tier": 4, "name": "fast_read", "max_iter": 5, |
92 | 92 | "prompt": "Read these three files as fast as possible: benchmark_data/explain_me.py, benchmark_data/buggy.py, benchmark_data/under_tested.py. Report their total combined file size in bytes and the first line of each file.\nOutput format:\nTOTAL_BYTES: <number>\nFILES:\n <filename>: <first line>\n <filename>: <first line>\n <filename>: <first line>", |
93 | | - "score": lambda out: score_speed_read(out, expected_bytes=parse_expected_sizes()), |
| 93 | + "score": lambda out, wt: score_speed_read(out, expected_bytes=parse_expected_sizes(), wall_time=wt), |
94 | 94 | }, |
95 | 95 | { |
96 | 96 | "id": "4.2", "tier": 4, "name": "quick_math", "max_iter": 5, |
97 | 97 | "prompt": "Using ONLY the shell tool (no Python), compute: 42 * 17, then add 256 to the result, then divide by 10. Report each intermediate result and the final answer.\nOutput format:\nSTEP1: 42 * 17 = <result>\nSTEP2: <result> + 256 = <result>\nSTEP3: <result> / 10 = <final>", |
98 | | - "score": lambda out: score_shell_math(out, expected=97), |
| 98 | + "score": lambda out, wt: score_shell_math(out, expected=97, wall_time=wt), |
99 | 99 | }, |
100 | 100 | { |
101 | 101 | "id": "4.3", "tier": 4, "name": "multi_search", "max_iter": 8, |
102 | 102 | "prompt": "In benchmark_data/, search for ALL of these patterns simultaneously (one call each, run in parallel):\n1. 'TODO' in all files\n2. 'def ' in all files\n3. 'import ' in all files\nReport the count of matches for each search.\nOutput format:\nTODO: <count>\nDEFS: <count>\nIMPORTS: <count>", |
103 | | - "score": lambda out: score_keywords(out, ["TODO:", "DEFS:", "IMPORTS:"], []), |
| 103 | + "score": lambda out, wt: score_multi_search(out, wall_time=wt), |
104 | 104 | }, |
105 | 105 | ] |
106 | 106 |
|
@@ -270,49 +270,87 @@ def verify_refactor() -> float: |
270 | 270 | return min(100, score) |
271 | 271 |
|
272 | 272 |
|
273 | | -def score_speed_read(output: str, expected_bytes: int) -> float: |
274 | | - """Score the fast_read task — check accuracy and reward speed. |
275 | | - v2.0: speed is measured by wall time (injected later).""" |
| 273 | +def score_speed_read(output: str, expected_bytes: int, wall_time: float = 120) -> float: |
| 274 | + """Format-tolerant fast_read scorer. |
| 275 | + Finds all numbers in output, picks closest to expected total bytes. |
| 276 | + 50% correctness (accuracy × 0.7 + file coverage × 0.3) + 50% speed bonus.""" |
276 | 277 | lower = output.lower() |
277 | | - # Must have total bytes |
278 | | - m = re.search(r"total_bytes:\s*(\d+)", lower) |
279 | | - if not m: |
| 278 | + numbers = [int(m.group()) for m in re.finditer(r'\d+', output)] |
| 279 | + if not numbers: |
280 | 280 | return 0.0 |
281 | 281 |
|
282 | | - reported = int(m.group(1)) |
283 | | - diff = abs(reported - expected_bytes) / max(expected_bytes, 1) |
284 | | - accuracy = max(0, 100 - diff * 200) |
| 282 | + # Accuracy: best match to expected total |
| 283 | + closest = min(numbers, key=lambda n: abs(n - expected_bytes)) |
| 284 | + accuracy = max(0, 1 - abs(closest - expected_bytes) / max(expected_bytes, 1)) |
| 285 | + |
| 286 | + # File coverage: mention all 3 files? |
| 287 | + files = sum(1 for f in ["explain_me.py", "buggy.py", "under_tested.py"] |
| 288 | + if f.lower() in lower) |
| 289 | + file_cov = files / 3 |
| 290 | + |
| 291 | + correctness = (accuracy * 0.7 + file_cov * 0.3) * 50 |
| 292 | + speed = _speed_bonus(wall_time) |
| 293 | + return round(min(100, correctness + speed), 1) |
| 294 | + |
| 295 | + |
| 296 | +def score_shell_math(output: str, expected: int, wall_time: float = 120) -> float: |
| 297 | + """Format-tolerant quick_math scorer. |
| 298 | + Looks for the final answer (97) anywhere in the output. |
| 299 | + 50% correctness + 50% speed bonus.""" |
| 300 | + # Can we find the final answer? |
| 301 | + numbers = [int(m.group()) for m in re.finditer(r'\d+', output)] |
| 302 | + found_final = expected in numbers |
| 303 | + # Also look for intermediate results |
| 304 | + found_714 = 714 in numbers |
| 305 | + found_970 = 970 in numbers |
| 306 | + |
| 307 | + correctness = 0.0 |
| 308 | + if found_final: |
| 309 | + correctness += 35 # final answer is most important |
| 310 | + if found_714: |
| 311 | + correctness += 10 # intermediate |
| 312 | + if found_970: |
| 313 | + correctness += 5 # intermediate |
| 314 | + |
| 315 | + speed = _speed_bonus(wall_time) |
| 316 | + return round(min(100, correctness + speed), 1) |
| 317 | + |
| 318 | + |
| 319 | +def score_multi_search(output: str, wall_time: float = 120) -> float: |
| 320 | + """Format-tolerant multi_search scorer. |
| 321 | + Looks for reasonable match counts in the output. |
| 322 | + Expected: TODO ~3, def ~66, import ~38. |
| 323 | + 50% correctness + 50% speed bonus.""" |
| 324 | + numbers = [int(m.group()) for m in re.finditer(r'\d+', output)] |
| 325 | + if not numbers: |
| 326 | + return 0.0 |
285 | 327 |
|
286 | | - # Must have 3 file names |
287 | | - files_found = sum(1 for f in ["explain_me.py", "buggy.py", "under_tested.py"] |
288 | | - if f.lower() in lower) |
289 | | - file_score = files_found / 3 * 30 |
| 328 | + # Score matches based on proximity to expected values |
| 329 | + def proximity(actual, expected): |
| 330 | + return max(0, 1 - abs(actual - expected) / max(expected, 1)) |
290 | 331 |
|
291 | | - return round(min(100, accuracy * 0.7 + file_score), 1) |
| 332 | + best_todo = max((proximity(n, 3) for n in numbers), default=0) |
| 333 | + best_def = max((proximity(n, 66) for n in numbers), default=0) |
| 334 | + best_import = max((proximity(n, 38) for n in numbers), default=0) |
292 | 335 |
|
| 336 | + # Also check that output mentions all three search terms |
| 337 | + mentions_todo = 'todo' in output.lower() or 'TODO' in output |
| 338 | + mentions_def = 'def' in output.lower() |
| 339 | + mentions_import = 'import' in output.lower() |
| 340 | + coverage = sum([mentions_todo, mentions_def, mentions_import]) / 3 |
293 | 341 |
|
294 | | -def score_shell_math(output: str, expected: int) -> float: |
295 | | - """Score the quick_math task — verify intermediate steps and final answer.""" |
296 | | - # Check for 3 steps |
297 | | - steps = len(re.findall(r"STEP\d", output, re.IGNORECASE)) |
298 | | - if steps < 3: |
299 | | - return steps * 15 # partial credit for individual steps |
| 342 | + correctness = (best_todo * 15 + best_def * 15 + best_import * 10 + coverage * 10) |
| 343 | + speed = _speed_bonus(wall_time) |
| 344 | + return round(min(100, correctness + speed), 1) |
300 | 345 |
|
301 | | - # Check final answer |
302 | | - m = re.search(r"STEP3.*?=\s*(\d+(?:\.\d+)?)", output, re.IGNORECASE) |
303 | | - if not m: |
304 | | - return 50.0 # steps done but no final answer |
305 | 346 |
|
306 | | - try: |
307 | | - final = float(m.group(1)) |
308 | | - if abs(final - expected) < 1: |
309 | | - return 100.0 |
310 | | - elif abs(final - expected) < 5: |
311 | | - return 70.0 |
312 | | - else: |
313 | | - return 50.0 |
314 | | - except ValueError: |
315 | | - return 50.0 |
| 347 | +def _speed_bonus(wall_time: float, max_score: float = 50.0) -> float: |
| 348 | + """Speed bonus: 0 at 120s, full at 10s, linear ramp.""" |
| 349 | + if wall_time <= 10: |
| 350 | + return max_score |
| 351 | + if wall_time >= 120: |
| 352 | + return 0.0 |
| 353 | + return round(max_score * (1 - (wall_time - 10) / 110), 1) |
316 | 354 |
|
317 | 355 |
|
318 | 356 | # ─── Benchmark Data ────────────────────────────────────────────────── |
@@ -606,7 +644,10 @@ def run_odek(task: dict) -> dict: |
606 | 644 | tokens_out = int(m.group(2)) |
607 | 645 |
|
608 | 646 | iterations = len(re.findall(r"═══ Iter \d+/\d+", combined)) |
609 | | - score = task["score"](combined) |
| 647 | + try: |
| 648 | + score = task["score"](combined, wall) |
| 649 | + except TypeError: |
| 650 | + score = task["score"](combined) |
610 | 651 |
|
611 | 652 | # v2.0: slow task penalty |
612 | 653 | if wall > SLOW_TASK_SECS and score > 0: |
@@ -637,7 +678,10 @@ def run_hermes(task: dict) -> dict: |
637 | 678 |
|
638 | 679 | wall = time.time() - start |
639 | 680 | output = (r.stdout or "") + (r.stderr or "") |
640 | | - score = task["score"](output) |
| 681 | + try: |
| 682 | + score = task["score"](output, wall) |
| 683 | + except TypeError: |
| 684 | + score = task["score"](output) |
641 | 685 |
|
642 | 686 | # v2.0: slow task penalty |
643 | 687 | if wall > SLOW_TASK_SECS and score > 0: |
|
0 commit comments