Skip to content

Commit 6b7cd3f

Browse files
committed
test: cover the max_turns-enrichment wiring seam + window-steer boundary (#600 N3/N4)
Follow-up coverage from the #600 review (feature approved as-is; tests only): - N3 (crit-8 wiring seam): the max_turns reason-enrichment was tested only at its two pure endpoints (stuck_guard.recent_failure_summary + the classifier on a hand-built string) — the append in _resolve_overall_task_status that joins them was unverified. Add TestMaxTurnsStuckEnrichment (monkeypatching hooks.last_stuck_summary): appends on error_max_turns, does NOT append on a non-max_turns error, leaves the reason unchanged when there's no summary, and does not double-append when the summary is already present. - N4 (crit-6 boundary): the window-steer was tested at 6/6 (steers) and 4/6 (doesn't) but not at the exact WINDOW_FAIL_THRESHOLD `>=` edge. Add test_window_steers_at_exactly_the_threshold (5/6 same-fingerprint fails in a full window → steers) and test_no_steer_when_window_not_yet_full (5 fails in a length-5 history → no steer, since _dominant_window_failure requires a full window). Imports WINDOW/WINDOW_FAIL_THRESHOLD so the constants' boundary semantics are now pinned. Full agent suite green (1218); ruff format + check clean.
1 parent e0efe30 commit 6b7cd3f

2 files changed

Lines changed: 84 additions & 0 deletions

File tree

agent/tests/test_pipeline_outcomes.py

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,3 +161,61 @@ def test_zero_turns_attempted_round_trips(self):
161161
# Zero is treated the same as None (falsy) so we don't clamp it to a
162162
# negative / nonsensical value.
163163
assert _compute_turns_completed("error_max_turns", 0, max_turns=10) == 0
164+
165+
166+
class TestMaxTurnsStuckEnrichment:
167+
"""N3 wiring seam (#600): _resolve_overall_task_status enriches a max_turns
168+
reason with the stuck-guard summary (hooks.last_stuck_summary). Previously
169+
tested only at its two pure endpoints; this drives the append itself."""
170+
171+
def test_max_turns_appends_stuck_summary(self, monkeypatch):
172+
import hooks
173+
174+
summary = "last tool calls repeated: git push — invalid credentials"
175+
monkeypatch.setattr(hooks, "last_stuck_summary", lambda: summary)
176+
ar = AgentResult(
177+
status="error_max_turns",
178+
error="Agent session error (subtype=error_max_turns)",
179+
)
180+
_, err = _resolve_overall_task_status(ar, build_ok=False, pr_url=None)
181+
assert err is not None
182+
assert "error_max_turns" in err
183+
assert summary in err
184+
185+
def test_non_max_turns_error_is_not_enriched(self, monkeypatch):
186+
# A generic failure must NOT pull in the stuck summary — only max_turns.
187+
import hooks
188+
189+
monkeypatch.setattr(hooks, "last_stuck_summary", lambda: "last tool calls repeated: X")
190+
ar = AgentResult(status="error", error="receive_response() failed: boom")
191+
_, err = _resolve_overall_task_status(ar, build_ok=False, pr_url=None)
192+
assert err is not None
193+
assert "last tool calls repeated" not in err
194+
195+
def test_max_turns_with_no_stuck_summary_left_unchanged(self, monkeypatch):
196+
# A task that used its turns productively (window not failure-dominated →
197+
# summary None) leaves the max_turns reason unchanged.
198+
import hooks
199+
200+
monkeypatch.setattr(hooks, "last_stuck_summary", lambda: None)
201+
ar = AgentResult(
202+
status="error_max_turns",
203+
error="Agent session error (subtype=error_max_turns)",
204+
)
205+
_, err = _resolve_overall_task_status(ar, build_ok=False, pr_url=None)
206+
assert err == "Agent session error (subtype=error_max_turns)"
207+
208+
def test_stuck_summary_not_double_appended(self, monkeypatch):
209+
# Idempotent: if the summary is already in the reason (e.g. a durable
210+
# re-resolution of the same result), it must not be appended twice.
211+
import hooks
212+
213+
summary = "last tool calls repeated: git push — invalid credentials"
214+
monkeypatch.setattr(hooks, "last_stuck_summary", lambda: summary)
215+
ar = AgentResult(
216+
status="error_max_turns",
217+
error=f"Agent session error (subtype=error_max_turns) — {summary}",
218+
)
219+
_, err = _resolve_overall_task_status(ar, build_ok=False, pr_url=None)
220+
assert err is not None
221+
assert err.count(summary) == 1

agent/tests/test_stuck_guard.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
from stuck_guard import (
66
STEER_THRESHOLD,
7+
WINDOW,
8+
WINDOW_FAIL_THRESHOLD,
79
StuckGuard,
810
_looks_failed,
911
_signature,
@@ -222,3 +224,27 @@ def test_healthy_iteration_below_window_threshold_no_steer(self):
222224
g.record_tool_result("Bash", {"command": f"cmd {i}"}, out)
223225
assert g.evaluate().kind == "none"
224226
assert g.recent_failure_summary() is None
227+
228+
def test_window_steers_at_exactly_the_threshold(self):
229+
# N4 boundary: exactly WINDOW_FAIL_THRESHOLD (5) same-fingerprint failures
230+
# in a FULL window of WINDOW (6) — the `>=` edge where an off-by-one would
231+
# hide. One OK dilutes the window to 5/6 fails, still == the threshold.
232+
assert WINDOW == 6 and WINDOW_FAIL_THRESHOLD == 5 # pin the constants
233+
g = StuckGuard()
234+
outcomes = [OK, _ERR, _ERR, _ERR, _ERR, _ERR] # 5 same-fp fails / full 6
235+
for i, out in enumerate(outcomes):
236+
g.record_tool_result("Bash", {"command": f"push {i}"}, out)
237+
action = g.evaluate()
238+
assert action.kind == "steer"
239+
assert action.signature == "__window__"
240+
assert g.recent_failure_summary() is not None
241+
242+
def test_no_steer_when_window_not_yet_full(self):
243+
# N4 boundary: WINDOW_FAIL_THRESHOLD failures but the window has fewer than
244+
# WINDOW entries — _dominant_window_failure requires a FULL window, so 5
245+
# identical failures in a length-5 history must NOT steer yet.
246+
g = StuckGuard()
247+
for i in range(WINDOW_FAIL_THRESHOLD): # 5 fails, window not yet at 6
248+
g.record_tool_result("Bash", {"command": f"push {i}"}, _ERR)
249+
assert g.evaluate().kind == "none"
250+
assert g.recent_failure_summary() is None

0 commit comments

Comments
 (0)