Skip to content

Commit 82ac931

Browse files
Trecekclaude
andauthored
Document stderr regex branch unreachability and add turn.failed fixture-driven tests (#4122)
## Summary Add a clarifying comment in `_exit_classification.py` documenting that the stderr regex branch (`_CODEX_CONTEXT_EXHAUSTION_PATTERN.search(result.stderr)`) is unreachable for Codex because `turn.failed` events arrive via stdout NDJSON, not stderr. Extend the `_turn_failed_ndjson` test helper with an optional `error_code` kwarg and add five new test methods exercising all `error.code` / `error.message` variants of `turn.failed` classification plus a stderr boundary test. ## Implementation Plan Plan file: `/home/talon/projects/autoskillit-runs/impl-20260627-014523-646887/.autoskillit/temp/make-plan/t5_p6_a14_wp1_stderr_regex_unreachable_plan_2026-06-27_015000.md` Closes #4050 🤖 Generated with [Claude Code](https://claude.com/claude-code) via AutoSkillit <!-- autoskillit:pipeline-signature steps=prepare_pr,run_arch_lenses,compose_pr,annotate_pr_diff,review_pr --> ## Token Usage Summary | Step | Model | count | uncached | output | cache_read | peak_ctx | turns | cache_write | time | |------|-------|-------|----------|--------|------------|----------|-------|-------------|------| | plan* | opus[1m] | 1 | 3.5k | 7.6k | 672.2k | 68.1k | 26 | 54.1k | 4m 17s | | verify* | sonnet | 1 | 70 | 7.1k | 390.3k | 70.7k | 23 | 52.2k | 4m 9s | | implement* | MiniMax-M3 | 1 | 62.7k | 6.2k | 1.3M | 0 | 68 | 0 | 2m 13s | | audit_impl* | sonnet | 1 | 1.6k | 6.3k | 159.7k | 38.4k | 14 | 27.7k | 2m 56s | | prepare_pr* | MiniMax-M3 | 1 | 40.1k | 1.7k | 110.1k | 0 | 11 | 0 | 38s | | compose_pr* | MiniMax-M3 | 1 | 36.5k | 1.5k | 249.1k | 0 | 14 | 0 | 55s | | **Total** | | | 144.5k | 30.2k | 2.9M | 70.7k | | 134.0k | 15m 10s | \* *Step used a non-Anthropic provider; caching behavior may differ.* ## Token Efficiency | Step | LoC Changed | cache_read/LoC | cache_write/LoC | output/LoC | |------|-------------|----------------|-----------------|------------| | plan | 0 | — | — | — | | verify | 0 | — | — | — | | implement | 49 | 27125.6 | 0.0 | 126.1 | | audit_impl | 0 | — | — | — | | prepare_pr | 0 | — | — | — | | compose_pr | 0 | — | — | — | | **Total** | **49** | 59398.6 | 2735.1 | 617.3 | ## Model Usage Breakdown | Model | steps | uncached | output | cache_read | cache_write | time | |-------|-------|----------|--------|------------|-------------|------| | opus[1m] | 1 | 3.5k | 7.6k | 672.2k | 54.1k | 4m 17s | | sonnet | 2 | 1.7k | 13.4k | 550.0k | 79.9k | 7m 6s | | MiniMax-M3 | 3 | 139.4k | 9.3k | 1.7M | 0 | 3m 46s | --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent cf41f0c commit 82ac931

2 files changed

Lines changed: 49 additions & 2 deletions

File tree

src/autoskillit/execution/session/_exit_classification.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ def classify_infra_exit(
115115
if capabilities is None or capabilities.supports_context_exhaustion_detection:
116116
if session._is_context_exhausted():
117117
return InfraExitCategory.CONTEXT_EXHAUSTED
118+
# Codex turn.failed arrives on stdout (NDJSON); this stderr branch is non-Codex fallback.
118119
if _CODEX_CONTEXT_EXHAUSTION_PATTERN.search(result.stderr):
119120
return InfraExitCategory.CONTEXT_EXHAUSTED
120121
# Rate limit detection — must precede all API_ERROR checks (including

tests/execution/test_exit_classification.py

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,11 @@ def _sr(
5656
)
5757

5858

59-
def _turn_failed_ndjson(error_message: str) -> str:
60-
return json.dumps({"type": "turn.failed", "error": {"message": error_message}})
59+
def _turn_failed_ndjson(error_message: str, *, error_code: str = "") -> str:
60+
error: dict[str, str] = {"message": error_message}
61+
if error_code:
62+
error["code"] = error_code
63+
return json.dumps({"type": "turn.failed", "error": error})
6164

6265

6366
class TestClassifyInfraExit:
@@ -412,6 +415,49 @@ def test_turn_failed_rate_limit_exceeded_jsonl_context_exhausted_false(self) ->
412415
adapted = _adapt_agent_result(agent_result)
413416
assert adapted.jsonl_context_exhausted is False
414417

418+
def test_turn_failed_error_code_only_classify_returns_context_exhausted(self) -> None:
419+
ndjson = _turn_failed_ndjson(
420+
"Agent terminated unexpectedly", error_code="context_length_exceeded"
421+
)
422+
agent_result = CodexResultParser().parse_stdout(ndjson)
423+
adapted = _adapt_agent_result(agent_result)
424+
result = _sr(returncode=1)
425+
assert classify_infra_exit(adapted, result) == InfraExitCategory.CONTEXT_EXHAUSTED
426+
427+
def test_turn_failed_error_code_and_message_classify_returns_context_exhausted(self) -> None:
428+
ndjson = _turn_failed_ndjson(
429+
"The model's context length has been exceeded",
430+
error_code="context_length_exceeded",
431+
)
432+
agent_result = CodexResultParser().parse_stdout(ndjson)
433+
adapted = _adapt_agent_result(agent_result)
434+
result = _sr(returncode=1)
435+
assert classify_infra_exit(adapted, result) == InfraExitCategory.CONTEXT_EXHAUSTED
436+
437+
def test_turn_failed_message_only_classify_returns_context_exhausted(self) -> None:
438+
ndjson = _turn_failed_ndjson("context_length_exceeded", error_code="")
439+
agent_result = CodexResultParser().parse_stdout(ndjson)
440+
adapted = _adapt_agent_result(agent_result)
441+
result = _sr(returncode=1)
442+
assert classify_infra_exit(adapted, result) == InfraExitCategory.CONTEXT_EXHAUSTED
443+
444+
def test_turn_failed_non_exhaustion_error_code_not_context_exhausted(self) -> None:
445+
ndjson = _turn_failed_ndjson("Internal server error", error_code="server_error")
446+
agent_result = CodexResultParser().parse_stdout(ndjson)
447+
adapted = _adapt_agent_result(agent_result)
448+
result = _sr(returncode=1)
449+
assert classify_infra_exit(adapted, result) == InfraExitCategory.API_ERROR
450+
451+
def test_stderr_non_exhaustion_not_context_exhausted(self) -> None:
452+
session = ClaudeSessionResult(
453+
subtype="success",
454+
is_error=False,
455+
result="",
456+
session_id="s1",
457+
)
458+
result = _sr(returncode=1, stderr="some transient error")
459+
assert classify_infra_exit(session, result) == InfraExitCategory.COMPLETED
460+
415461

416462
class TestRateLimitClassification:
417463
def test_rate_limited_text_classified_as_rate_limited(self) -> None:

0 commit comments

Comments
 (0)