Skip to content

Commit 2837130

Browse files
committed
fix(errors): address #599 nits N1-N3 — [auto-retried] breadcrumb + test seams
N1 (Medium): the [auto-retried] marker was never stamped on branch 4 (transient- then-transient), because startSessionWithRetry THROWS there and the caller's autoRetried flag is only set on the success paths. A double-transient failure was told 'reply to retry' instead of 'I already retried' — the exact confusion the marker exists to prevent. Fix: tag the thrown error with a Symbol (AUTO_RETRIED) + export isAutoRetried(); orchestrate-task reads it in the catch and stamps the marker on both paths a retry ran. Corrected the docstring (was claiming the fact was 'observable via the thrown-from state' — it wasn't). N2 (Low): pin the two untested wiring seams. CDK: branch-4 test now asserts isAutoRetried(err) is true (and false for a first-attempt/non-object failure). Agent: new tests drive _stuck_guard_between_turns_hook end-to-end and assert the _LAST_STUCK_SUMMARY latch is written (and cleared on a healthy window) — the production path the enrichment test's monkeypatch bypasses. Deleting hooks.py's latch write now fails a test. N3 (Low): pin retryGuidance's two USER fall-through branches (retryable-user non-guardrail, not-retryable-user) so the #247 failure-renderer copy contract can't rot. Full cdk (2345) + agent (1251) gates green.
1 parent 4e38a45 commit 2837130

5 files changed

Lines changed: 143 additions & 11 deletions

File tree

agent/tests/test_hooks.py

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,14 @@
99

1010
from hooks import (
1111
_reset_blocker_reason_for_tests,
12+
_stuck_guard_between_turns_hook,
1213
build_hook_matchers,
1314
detect_egress_denial,
1415
last_blocker_reason,
16+
last_stuck_summary,
1517
post_tool_use_hook,
1618
pre_tool_use_hook,
19+
reset_stuck_summary,
1720
)
1821
from policy import PolicyEngine
1922

@@ -1721,6 +1724,53 @@ def test_post_tool_use_records_failures_into_the_guard(self):
17211724
# the guard now has enough failures to steer
17221725
assert guard.evaluate().kind == "steer"
17231726

1727+
def test_between_turns_hook_latches_stuck_summary_from_the_guard(self):
1728+
# #599 N2: pin the PRODUCTION write path for the _LAST_STUCK_SUMMARY latch
1729+
# (hooks.py:1464-1465). The enrichment tests monkeypatch the getter, so
1730+
# without this test deleting those two lines would leave the latch
1731+
# permanently None and every test would still pass. Drive the real hook
1732+
# with a failure-dominated guard and assert the module latch is populated.
1733+
from stuck_guard import WINDOW, StuckGuard
1734+
1735+
reset_stuck_summary()
1736+
assert last_stuck_summary() is None
1737+
guard = StuckGuard()
1738+
cmd = {"command": "mise //cdk:test"}
1739+
# recent_failure_summary needs a FULL window (>= WINDOW) of byte-identical
1740+
# failures (WINDOW_FAIL_THRESHOLD of them) — fill it past WINDOW.
1741+
for _ in range(WINDOW + 2):
1742+
guard.record_tool_result("Bash", cmd, self._oom())
1743+
# Precondition: the guard itself considers the window failure-dominated.
1744+
assert guard.recent_failure_summary() is not None
1745+
1746+
result = _stuck_guard_between_turns_hook({"stuck_guard": guard})
1747+
# advisory steer text returned…
1748+
assert isinstance(result, list)
1749+
# …and, crucially, the terminal-reason latch was written by the hook.
1750+
summary = last_stuck_summary()
1751+
assert summary is not None
1752+
assert "last tool calls repeated" in summary
1753+
reset_stuck_summary()
1754+
1755+
def test_between_turns_hook_clears_stuck_summary_when_not_failure_dominated(self):
1756+
# Symmetric guard: a recovered task (clean window) must CLEAR the latch,
1757+
# not leave a stale "stuck" summary that a later max_turns cap would echo.
1758+
# Pre-seed a stale latch via a failure-dominated guard.
1759+
from stuck_guard import WINDOW, StuckGuard
1760+
1761+
stuck = StuckGuard()
1762+
for _ in range(WINDOW + 2):
1763+
stuck.record_tool_result("Bash", {"command": "mise //cdk:test"}, self._oom())
1764+
_stuck_guard_between_turns_hook({"stuck_guard": stuck})
1765+
assert last_stuck_summary() is not None
1766+
1767+
# A fresh, healthy guard on the next turn clears it (recent_failure_summary None).
1768+
healthy = StuckGuard()
1769+
healthy.record_tool_result("Read", {"file": "a.py"}, "def hello(): return 1")
1770+
_stuck_guard_between_turns_hook({"stuck_guard": healthy})
1771+
assert last_stuck_summary() is None
1772+
reset_stuck_summary()
1773+
17241774
def test_post_tool_use_record_error_never_blocks_screening(self):
17251775
# A guard that raises on record must not break the PASS_THROUGH path.
17261776
from stuck_guard import StuckGuard

cdk/src/handlers/orchestrate-task.ts

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ import {
3737
type PollState,
3838
} from './shared/orchestrator';
3939
import { runPreflightChecks } from './shared/preflight';
40-
import { startSessionWithRetry } from './shared/session-start-retry';
40+
import { isAutoRetried, startSessionWithRetry } from './shared/session-start-retry';
4141
import { deleteEcsPayload } from './shared/strategies/ecs-strategy';
4242
import type { TaskRecord } from './shared/types';
4343
import { workflowIsReadOnly, workflowRequiresRepo } from './shared/workflows';
@@ -217,9 +217,15 @@ const durableHandler: DurableExecutionHandler<OrchestrateTaskEvent, void> = asyn
217217
// classification). It is a breadcrumb for a FORTHCOMING failure renderer to
218218
// detect and surface "I already tried again" to the channel — no consumer
219219
// renders it yet on this branch (retryGuidance() in error-classifier.ts is
220-
// the intended copy source; it ships ahead of its consumer). Only stamped
221-
// when the single transient retry above also failed.
222-
const retriedNote = autoRetried ? ' [auto-retried]' : '';
220+
// the intended copy source; it ships ahead of its consumer).
221+
//
222+
// Stamp on BOTH paths a retry ran (#599 N1): branch 3 sets `autoRetried`
223+
// above then a LATER step throws; branch 4 (transient-then-transient) throws
224+
// FROM startSessionWithRetry before `autoRetried` is assigned, so the local
225+
// is still false — read the fact off the thrown error via isAutoRetried().
226+
// Without this, a double-transient failure was told "reply to retry" instead
227+
// of "I already retried" — the exact confusion the marker exists to prevent.
228+
const retriedNote = (autoRetried || isAutoRetried(err)) ? ' [auto-retried]' : '';
223229
await failTask(taskId, TaskStatus.HYDRATING, `Session start failed: ${String(err)}${retriedNote}`, task.user_id, true, task.repo);
224230
throw err;
225231
}

cdk/src/handlers/shared/session-start-retry.ts

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,20 @@ export interface StartSessionWithRetryResult {
5454
readonly autoRetried: boolean;
5555
}
5656

57+
/**
58+
* Symbol tag pinned onto the error thrown from branch 4 (transient-then-transient)
59+
* so the retry fact survives the throw — the ``autoRetried`` result field only
60+
* exists on the success paths. Kept as a Symbol (not an own string prop) so it
61+
* never collides with, or leaks into, the error's serialized shape. Read via
62+
* {@link isAutoRetried}.
63+
*/
64+
const AUTO_RETRIED = Symbol('autoRetried');
65+
66+
/** True iff ``err`` was thrown after an auto-retry already ran (branch 4). */
67+
export function isAutoRetried(err: unknown): boolean {
68+
return typeof err === 'object' && err !== null && (err as Record<symbol, unknown>)[AUTO_RETRIED] === true;
69+
}
70+
5771
/**
5872
* Start a compute session, auto-retrying ONCE on a transient failure.
5973
*
@@ -62,9 +76,13 @@ export interface StartSessionWithRetryResult {
6276
* 2. first attempt fails NON-transient → re-throw the original error (no retry).
6377
* 3. first attempt fails transient, retry succeeds → return it, ``autoRetried: true``.
6478
* 4. first attempt fails transient, retry also fails → throw the retry's error
65-
* (with ``autoRetried`` observable via the thrown-from state; the caller
66-
* stamps its own ``[auto-retried]`` marker — it owns ``autoRetried`` because
67-
* this function throws rather than returns on the double-failure).
79+
* TAGGED with ``autoRetried = true`` (see {@link isAutoRetried}). The result
80+
* object carries ``autoRetried`` only on the success paths (1/3); on the
81+
* double-failure this function throws, so the fact is instead pinned onto the
82+
* thrown error itself — the caller reads it via {@link isAutoRetried} to stamp
83+
* the ``[auto-retried]`` marker. Without the tag the caller could not tell a
84+
* double-transient failure (retry ran) from a first-attempt failure (it did
85+
* not), and would wrongly tell the user "reply to retry" (N1).
6886
*
6987
* The ``emitRetryEvent`` call is BEST-EFFORT and internally guarded (B1): a
7088
* TaskEvents PutItem fault (throttle/timeout — exactly the conditions that
@@ -113,7 +131,18 @@ export async function startSessionWithRetry(
113131
error: emitErr instanceof Error ? emitErr.message : String(emitErr),
114132
});
115133
}
116-
const handle = await strategy.startSession(input);
117-
return { handle, autoRetried: true };
134+
try {
135+
const handle = await strategy.startSession(input);
136+
return { handle, autoRetried: true };
137+
} catch (retryErr) {
138+
// Branch 4: the retry ALSO failed. Pin the retry fact onto the thrown error
139+
// so the caller can stamp ``[auto-retried]`` (N1) — otherwise a double-
140+
// transient failure is indistinguishable from a first-attempt failure and
141+
// the user is wrongly told "reply to retry" instead of "I already retried".
142+
if (typeof retryErr === 'object' && retryErr !== null) {
143+
(retryErr as Record<symbol, unknown>)[AUTO_RETRIED] = true;
144+
}
145+
throw retryErr;
146+
}
118147
}
119148
}

cdk/test/handlers/shared/error-classifier.test.ts

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -600,6 +600,39 @@ describe('classifyError', () => {
600600
expect(g).toMatch(/edit the request/i);
601601
});
602602

603+
// #599 N3: pin the two USER fall-through branches so the #247 failure-renderer
604+
// contract can't rot silently. Built as explicit classifications (the exact
605+
// category/errorClass/retryable each branch keys on) rather than relying on a
606+
// sample string that might reclassify later.
607+
test('retryGuidance: retryable USER (non-guardrail) → "reply here with any extra guidance"', () => {
608+
const cls: ErrorClassification = {
609+
category: ErrorCategory.AGENT,
610+
title: 'build failed',
611+
description: 'the build/test step failed',
612+
remedy: 'fix the failing step',
613+
retryable: true,
614+
errorClass: ErrorClass.USER,
615+
};
616+
const g = retryGuidance(cls);
617+
expect(g).toMatch(/extra guidance/i);
618+
expect(g).toMatch(/try again/i);
619+
expect(g).not.toMatch(/edit the request/i); // not the guardrail branch
620+
});
621+
622+
test('retryGuidance: not-retryable USER/unknown → "a retry may not resolve this"', () => {
623+
const cls: ErrorClassification = {
624+
category: ErrorCategory.UNKNOWN,
625+
title: 'agent reported non-success',
626+
description: 'the agent finished without success',
627+
remedy: 'review the task output',
628+
retryable: false,
629+
errorClass: ErrorClass.USER,
630+
};
631+
const g = retryGuidance(cls);
632+
expect(g).toMatch(/may not resolve this/i);
633+
expect(g).toMatch(/contact your ABCA admin/i);
634+
});
635+
603636
test('isTransientError is false for null / absent classification', () => {
604637
expect(isTransientError(null)).toBe(false);
605638
expect(isTransientError(undefined)).toBe(false);

cdk/test/handlers/shared/session-start-retry.test.ts

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
*/
1919

2020
import type { SessionHandle } from '../../../src/handlers/shared/compute-strategy';
21-
import { startSessionWithRetry } from '../../../src/handlers/shared/session-start-retry';
21+
import { isAutoRetried, startSessionWithRetry } from '../../../src/handlers/shared/session-start-retry';
2222

2323
const HANDLE: SessionHandle = {
2424
sessionId: 'sess-1',
@@ -79,15 +79,29 @@ describe('startSessionWithRetry — the 4 branches (#599 B2)', () => {
7979
expect(emitReasons).toHaveLength(1); // the session_start_retry event fired once
8080
});
8181

82-
it('4. transient then transient → throws the retry error (second failure surfaces)', async () => {
82+
it('4. transient then transient → throws the retry error TAGGED autoRetried (#599 N1)', async () => {
8383
const secondErr = new Error('TaskDefinition is inactive (again)');
8484
const startSession = jest
8585
.fn()
8686
.mockRejectedValueOnce(TRANSIENT)
8787
.mockRejectedValueOnce(secondErr);
8888
const { d } = deps();
89+
// The double-transient error must carry the retry fact across the throw so the
90+
// caller stamps `[auto-retried]` (a first-attempt failure must NOT be tagged).
8991
await expect(startSessionWithRetry({ startSession }, {} as never, d)).rejects.toBe(secondErr);
9092
expect(startSession).toHaveBeenCalledTimes(2);
93+
expect(isAutoRetried(secondErr)).toBe(true);
94+
});
95+
96+
it('isAutoRetried is false for a first-attempt (non-transient) failure and non-objects', async () => {
97+
// Branch 2's re-thrown error ran no retry → must NOT be tagged, else the caller
98+
// would wrongly tell the user "I already retried".
99+
const startSession = jest.fn().mockRejectedValueOnce(NON_TRANSIENT);
100+
const { d } = deps();
101+
await expect(startSessionWithRetry({ startSession }, {} as never, d)).rejects.toBe(NON_TRANSIENT);
102+
expect(isAutoRetried(NON_TRANSIENT)).toBe(false);
103+
expect(isAutoRetried(undefined)).toBe(false);
104+
expect(isAutoRetried('a string error')).toBe(false);
91105
});
92106
});
93107

0 commit comments

Comments
 (0)