Skip to content

Commit 7ef6458

Browse files
rasdaniclaude
rasdani
and
claude
authored
feat(v1): add capture_patch/resolve_head git utils for SWE tasksets (#2054)
* feat(v1): add capture_patch/resolve_head git utils for SWE tasksets Upstreams the patch-capture helper duplicated across the research-environments SWE tasksets (research-environments#674): capture_patch snapshots the agent's cumulative diff into trace.info["patch"] at finalize time (2 MB cap with patch_truncated; failures record patch_error instead of failing the rollout; index always reset; add/diff exit codes both surfaced), and resolve_head records the pre-agent SHA at setup so agent commits stay inside the diff. Exported from verifiers.v1 so tasksets import one canonical implementation. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(v1): fail patch capture when git reset fails The reset's exit status was discarded, so a reset failure after a clean add and diff (e.g. ENOSPC rewriting .git/index) recorded a successful patch while leaving the tree staged for later scoring to trip on. Every mutating step now reports: capture fails with the first failing of add, reset, or diff. Covered by a git-shim test that fails only reset. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * chore(v1): remove the full diff file once the capped copy exists The unbounded /tmp/vf_agent_patch_full is only needed until head produces the capped copy; delete it inside the capture sequence so it never outlives the runtime.run call — on the subprocess runtime those paths are the host's /tmp. The capped file (<=2MB+1) is read back by capture_patch after the command returns, so it cannot be removed in-sequence. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(v1): surface head failures in patch capture head was the last unchecked step in the capture sequence: the redirect truncates the capped file before head runs, so a failed head (ENOSPC, I/O error) could read back as a silently empty patch with exit 0. Fold head's exit code into the precedence chain (add > reset > diff > head) so every step reports, and cover it with a head-shim test. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(v1): per-invocation temp paths and cleanup for patch capture Fixed /tmp names let concurrent rollouts on shared-filesystem runtimes (subprocess) read each other's patches between the write and the read — silent cross-rollout contamination — and let an agent pre-create the predictable path as a FIFO so the shell redirect blocks the rollout forever. Suffix both paths with a host-generated nonce per invocation, pre-remove the targets before the redirect opens, and best-effort remove both files after the read since unique names no longer overwrite each other. Covered by a path-uniqueness + cleanup test. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * chore(v1): remove patch capture test file --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
1 parent 01ee452 commit 7ef6458

2 files changed

Lines changed: 122 additions & 0 deletions

File tree

‎verifiers/v1/__init__.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@
7272
)
7373
from verifiers.v1.retries import RetryConfig, RolloutRetryConfig
7474
from verifiers.v1.rollout import Rollout
75+
from verifiers.v1.utils.git import (
76+
PATCH_CAP_BYTES as PATCH_CAP_BYTES,
77+
capture_patch as capture_patch,
78+
resolve_head as resolve_head,
79+
)
7580
from verifiers.v1.runtimes import (
7681
DockerConfig,
7782
PrimeConfig,
@@ -260,6 +265,10 @@
260265
"RubricJudge",
261266
"RubricJudgeConfig",
262267
"Criterion",
268+
# git patch capture
269+
"PATCH_CAP_BYTES",
270+
"capture_patch",
271+
"resolve_head",
263272
# scoring
264273
"compare_stdout_results",
265274
"extract_boxed_answer",

‎verifiers/v1/utils/git.py‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
"""Persist an agent's final git patch into `trace.info` at finalize time.
2+
3+
SWE-style tasksets call `capture_patch` from `Task.finalize` — after the harness
4+
finishes, while the runtime is live, before scoring mutates the repo (restoring
5+
test files, switching commits) — so the diff is exactly what the agent produced,
6+
including edits to test files (intentional: they reveal reward hacking).
7+
8+
The diff is taken against `base_commit` when the caller has one — a dataset row
9+
field, or a SHA recorded with `resolve_head` at setup time and kept in host
10+
memory (never the sandbox, where an agent could tamper with it) — so commits the
11+
agent made are included. Bare `HEAD` is the fallback of last resort and misses
12+
agent commits.
13+
"""
14+
15+
from __future__ import annotations
16+
17+
import uuid
18+
from typing import TYPE_CHECKING
19+
20+
if TYPE_CHECKING:
21+
from verifiers.v1.runtimes import Runtime
22+
from verifiers.v1.trace import Trace
23+
24+
PATCH_CAP_BYTES = 2_000_000
25+
"""Truncate captured patches beyond this size; `info["patch_truncated"]` marks it."""
26+
27+
# Temp paths are suffixed per invocation with a host-generated nonce: fixed names
28+
# would let concurrent rollouts on a shared-filesystem runtime (subprocess) read
29+
# each other's patches between the write and the read, and would let an agent
30+
# pre-create the predictable path as a FIFO so the shell's redirect blocks the
31+
# rollout forever.
32+
_FULL = "/tmp/vf_agent_patch_full"
33+
_CAPPED = "/tmp/vf_agent_patch"
34+
35+
# `git reset -q` must run even when staging or diffing fails, or the error path
36+
# leaves the tree staged and can break scoring's later checkouts. Every step
37+
# reports: a failure in add, diff, reset, or head fails the capture. A failed
38+
# add (e.g. a stale index.lock from a killed agent git command) leaves a stale
39+
# index, so letting the diff's clean exit stand would record a silently
40+
# incomplete patch; a failed reset (e.g. ENOSPC rewriting .git/index) leaves the
41+
# tree staged, so reporting success would hide a state later scoring may trip
42+
# on; a failed head leaves an empty {capped} (the redirect truncates it before
43+
# head runs), which would read back as a silently empty patch.
44+
_DIFF = (
45+
"rm -f {full} {capped}; "
46+
"git add -A; "
47+
"add_rc=$?; "
48+
'git -c core.quotepath=off diff --cached --binary "$VF_DIFF_BASE" > {full}; '
49+
"diff_rc=$?; "
50+
"git reset -q; "
51+
"reset_rc=$?; "
52+
"head -c {cap} {full} > {capped}; "
53+
"head_rc=$?; "
54+
"rm -f {full}; "
55+
"rc=$head_rc; "
56+
'[ "$diff_rc" -ne 0 ] && rc=$diff_rc; '
57+
'[ "$reset_rc" -ne 0 ] && rc=$reset_rc; '
58+
'[ "$add_rc" -ne 0 ] && rc=$add_rc; '
59+
'exit "$rc"'
60+
)
61+
62+
63+
async def resolve_head(runtime: Runtime, env: dict | None = None) -> str:
64+
"""The repo's current commit SHA, or "" when unresolvable.
65+
66+
Call at the end of `setup`, before the agent runs, and keep the result in
67+
host memory (e.g. a dict on the task keyed by `id(runtime)`) for `finalize`
68+
to pass as `base_commit` — diffing against a pre-agent SHA is what keeps
69+
commits the agent makes inside the captured patch.
70+
"""
71+
result = await runtime.run(["git", "rev-parse", "HEAD"], env or {})
72+
if result.exit_code != 0:
73+
return ""
74+
return (result.stdout or "").strip()
75+
76+
77+
async def capture_patch(
78+
trace: Trace, runtime: Runtime, base_commit: str = "", env: dict | None = None
79+
) -> None:
80+
"""Snapshot the agent's cumulative diff into `trace.info["patch"]`.
81+
82+
Best-effort by design: a rollout whose sandbox died or whose repo state is
83+
broken records `info["patch_error"]` instead of failing the rollout —
84+
scoring still runs and the error stays visible in results.
85+
"""
86+
nonce = uuid.uuid4().hex
87+
full, capped = f"{_FULL}_{nonce}", f"{_CAPPED}_{nonce}"
88+
cmd = _DIFF.format(full=full, capped=capped, cap=PATCH_CAP_BYTES + 1)
89+
try:
90+
result = await runtime.run(
91+
["sh", "-c", cmd],
92+
{**(env or {}), "VF_DIFF_BASE": base_commit or "HEAD"},
93+
)
94+
if result.exit_code != 0:
95+
trace.info["patch_error"] = (
96+
f"exit={result.exit_code} {(result.stderr or '').strip()[-500:]}"
97+
)
98+
return
99+
raw = await runtime.read(capped)
100+
except Exception as exc: # noqa: BLE001 - capture must never fail the rollout.
101+
trace.info["patch_error"] = f"{type(exc).__name__}: {exc}"
102+
return
103+
finally:
104+
# Unique names don't overwrite each other, so leftovers would accumulate
105+
# on shared-filesystem runtimes; removal is best-effort by design.
106+
try:
107+
await runtime.run(["rm", "-f", full, capped], env or {})
108+
except Exception: # noqa: BLE001,S110 - cleanup must never fail the rollout.
109+
pass
110+
if len(raw) > PATCH_CAP_BYTES:
111+
raw = raw[:PATCH_CAP_BYTES]
112+
trace.info["patch_truncated"] = True
113+
trace.info["patch"] = raw.decode("utf-8", errors="replace")

0 commit comments

Comments
 (0)