-
Notifications
You must be signed in to change notification settings - Fork 36
Expand file tree
/
Copy patherror-classifier.test.ts
More file actions
782 lines (684 loc) · 35.3 KB
/
Copy patherror-classifier.test.ts
File metadata and controls
782 lines (684 loc) · 35.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
/**
* MIT No Attribution
*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of
* the Software without restriction, including without limitation the rights to
* use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
* the Software, and to permit persons to whom the Software is furnished to do so.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
import { classifyError, ErrorCategory, ErrorClass, isTransientError, retryGuidance, type ErrorClassification } from '../../../src/handlers/shared/error-classifier';
import { toTaskDetail, type TaskRecord } from '../../../src/handlers/shared/types';
describe('classifyError', () => {
// --- Null / empty inputs ---
test('returns null for undefined', () => {
expect(classifyError(undefined)).toBeNull();
});
test('returns null for null', () => {
expect(classifyError(null)).toBeNull();
});
test('returns null for empty string', () => {
expect(classifyError('')).toBeNull();
});
// --- Auth errors ---
describe('auth errors', () => {
test('classifies insufficient GitHub repo permissions (preflight)', () => {
const result = classifyError(
'Pre-flight check failed: INSUFFICIENT_GITHUB_REPO_PERMISSIONS — Token cannot push to owner/repo.',
);
expect(result!.category).toBe(ErrorCategory.AUTH);
expect(result!.title).toBe('Insufficient GitHub permissions');
expect(result!.retryable).toBe(false);
});
test('classifies repo not found or no access', () => {
const result = classifyError(
'Pre-flight check failed: REPO_NOT_FOUND_OR_NO_ACCESS — GitHub API returned HTTP 404 for owner/repo',
);
expect(result!.category).toBe(ErrorCategory.AUTH);
expect(result!.title).toBe('Repository not found or inaccessible');
expect(result!.retryable).toBe(false);
});
test('classifies PR not found or closed', () => {
const result = classifyError(
'Pre-flight check failed: PR_NOT_FOUND_OR_CLOSED — PR #42 in owner/repo is closed, not open',
);
expect(result!.category).toBe(ErrorCategory.AUTH);
expect(result!.title).toBe('Pull request not found or closed');
expect(result!.retryable).toBe(false);
});
test('classifies token push scope error (detailed message)', () => {
const result = classifyError(
'Pre-flight check failed: INSUFFICIENT_GITHUB_REPO_PERMISSIONS — Token cannot push to owner/repo. Required: push. For fine-grained PATs use Contents Read and write.',
);
expect(result!.category).toBe(ErrorCategory.AUTH);
expect(result!.retryable).toBe(false);
});
test('classifies token PR scope error', () => {
const result = classifyError(
'Pre-flight check failed: INSUFFICIENT_GITHUB_REPO_PERMISSIONS — Token cannot interact with pull requests on owner/repo.',
);
expect(result!.category).toBe(ErrorCategory.AUTH);
expect(result!.retryable).toBe(false);
});
test('classifies bare "Token cannot push to" without error code', () => {
const result = classifyError('Token cannot push to owner/repo');
expect(result!.category).toBe(ErrorCategory.AUTH);
expect(result!.title).toBe('Insufficient GitHub token scopes');
});
test('classifies bare "Token cannot interact with pull requests on" without error code', () => {
const result = classifyError('Token cannot interact with pull requests on owner/repo');
expect(result!.category).toBe(ErrorCategory.AUTH);
expect(result!.title).toBe('Insufficient GitHub token scopes');
});
});
// --- Network errors ---
describe('network errors', () => {
test('classifies GitHub unreachable', () => {
const result = classifyError(
'Pre-flight check failed: GITHUB_UNREACHABLE — connect ETIMEDOUT 140.82.121.6:443',
);
expect(result!.category).toBe(ErrorCategory.NETWORK);
expect(result!.title).toBe('GitHub API unreachable');
expect(result!.retryable).toBe(true);
});
test('classifies GitHub API HTTP 5xx', () => {
const result = classifyError(
'Pre-flight check failed: GITHUB_UNREACHABLE — GitHub API returned HTTP 503',
);
expect(result!.category).toBe(ErrorCategory.NETWORK);
expect(result!.retryable).toBe(true);
});
test('classifies GitHub API HTTP 4xx in preflight detail', () => {
const result = classifyError(
'Pre-flight check failed: GITHUB_UNREACHABLE — GitHub API returned HTTP 403 for owner/repo',
);
expect(result!.category).toBe(ErrorCategory.NETWORK);
expect(result!.retryable).toBe(true);
});
test('classifies bare GitHub API HTTP status without GITHUB_UNREACHABLE', () => {
const result = classifyError('GitHub API returned HTTP 502 during polling');
expect(result!.category).toBe(ErrorCategory.NETWORK);
expect(result!.title).toBe('GitHub API error');
});
});
// --- Concurrency errors ---
describe('concurrency errors', () => {
test('classifies user concurrency limit', () => {
const result = classifyError('User concurrency limit reached');
expect(result!.category).toBe(ErrorCategory.CONCURRENCY);
expect(result!.title).toBe('Concurrency limit reached');
expect(result!.retryable).toBe(true);
});
});
// --- Compute errors ---
describe('compute errors', () => {
test('classifies session start failure', () => {
const result = classifyError('Session start failed: ServiceQuotaExceededException');
expect(result!.category).toBe(ErrorCategory.COMPUTE);
expect(result!.title).toBe('Agent session failed to start');
expect(result!.retryable).toBe(true);
});
test('classifies ECS container failure', () => {
const result = classifyError('ECS container failed: OutOfMemoryError');
expect(result!.category).toBe(ErrorCategory.COMPUTE);
expect(result!.title).toBe('ECS container failed');
expect(result!.retryable).toBe(true);
});
test('classifies claude Exec-format / broken-shim as a transient image issue (ABCA-659, not "Unexpected error")', () => {
// The raw run_agent failure the broken agent image produced.
const result = classifyError(
"Workflow run_agent step failed: OSError: [Errno 8] Exec format error: 'claude'",
);
expect(result!.category).toBe(ErrorCategory.COMPUTE);
expect(result!.title).toBe('Couldn\'t start the coding agent (environment issue)');
expect(result!.retryable).toBe(true);
// MUST be transient so retryGuidance tells the user to just reply-to-retry
// (and escalate to an admin only if it persists) — not the bare
// "Unexpected error" with no guidance it used to fall through to.
expect(result!.errorClass).toBe(ErrorClass.TRANSIENT);
expect(result!.remedy).toMatch(/try again|rebuild|admin/i);
});
test('classifies the claude shim self-report ("native binary not installed")', () => {
const result = classifyError('Error: claude native binary not installed.');
expect(result!.category).toBe(ErrorCategory.COMPUTE);
expect(result!.errorClass).toBe(ErrorClass.TRANSIENT);
});
test('classifies ECS exit without terminal status', () => {
const result = classifyError(
'ECS task exited successfully but agent never wrote terminal status after 5 polls',
);
expect(result!.category).toBe(ErrorCategory.COMPUTE);
expect(result!.title).toBe('Agent exited without reporting status');
expect(result!.retryable).toBe(true);
});
test('classifies ECS poll failures', () => {
const result = classifyError(
'ECS poll failed 3 consecutive times: AccessDeniedException',
);
expect(result!.category).toBe(ErrorCategory.COMPUTE);
expect(result!.title).toBe('ECS polling failure');
expect(result!.retryable).toBe(true);
});
test('classifies session never started (HYDRATING timeout)', () => {
const result = classifyError(
'Session never started — poll timeout exceeded while still HYDRATING',
);
expect(result!.category).toBe(ErrorCategory.COMPUTE);
expect(result!.title).toBe('Agent session never started');
expect(result!.retryable).toBe(true);
});
test('classifies agent heartbeat loss', () => {
const result = classifyError(
'Agent session lost: no recent heartbeat from the runtime (container may have crashed, been OOM-killed, or stopped)',
);
expect(result!.category).toBe(ErrorCategory.COMPUTE);
expect(result!.title).toBe('Agent session lost');
expect(result!.retryable).toBe(true);
});
});
// --- Agent errors ---
describe('agent errors', () => {
test('classifies SDK stream ended without ResultMessage', () => {
const result = classifyError(
'Agent SDK stream ended without a ResultMessage (agent_status=unknown). Treat as failure: possible SDK bug, network interruption, or protocol mismatch.',
);
expect(result!.category).toBe(ErrorCategory.AGENT);
expect(result!.title).toBe('Agent SDK stream ended unexpectedly');
expect(result!.retryable).toBe(true);
});
test('classifies SDK stream ended with chained error', () => {
const result = classifyError(
'some prior error; Agent SDK stream ended without a ResultMessage (agent_status=unknown). Treat as failure.',
);
expect(result!.category).toBe(ErrorCategory.AGENT);
expect(result!.retryable).toBe(true);
});
test('classifies task did not succeed', () => {
const result = classifyError(
"Task did not succeed (agent_status='error', build_ok=False)",
);
expect(result!.category).toBe(ErrorCategory.AGENT);
expect(result!.title).toBe('Agent task did not succeed');
expect(result!.retryable).toBe(false);
});
test('classifies error_max_turns as TIMEOUT with specific title (ordered before generic catch-all)', () => {
// Regression guard: pre-fix, the agent's specific
// ``agent_status='error_max_turns'`` signal was swallowed by the
// generic "Agent task did not succeed" title, leaving users
// without a clear remedy. The specific pattern must match first.
const result = classifyError(
"Task did not succeed (agent_status='error_max_turns', build_ok=False)",
);
expect(result!.category).toBe(ErrorCategory.TIMEOUT);
expect(result!.title).toBe('Exceeded max turns');
expect(result!.retryable).toBe(true);
expect(result!.remedy).toMatch(/--max-turns/);
});
test('ABCA-662: max_turns with an observed repeated failure stays "Exceeded max turns" and makes NO causal claim', () => {
// When the agent capped out with the last several calls being the same
// repeated failure, the pipeline appends a NEUTRAL observation ("last tool
// calls repeated: …"). The classification must NOT re-title the failure as
// "retrying a failing step" or assert more turns wouldn't help — the window
// (last few calls) can't tell a hard blocker from a long task that hit a
// recoverable snag late (662: siblings pushed fine → transient). It stays the
// plain max_turns bucket; the observed detail rides along in the message.
const result = classifyError(
"Agent session error (subtype='error_max_turns') — last tool calls repeated: "
+ '`git push --force-with-lease` — remote: invalid credentials fatal: exit 128',
);
expect(result!.category).toBe(ErrorCategory.TIMEOUT);
expect(result!.title).toBe('Exceeded max turns');
expect(result!.retryable).toBe(true);
// Does not editorialize: no "spinning" / "won't help" claim. It points the
// reader at the detail and still offers the environment-blocker path.
expect(result!.title).not.toMatch(/retrying a failing step/i);
expect(result!.remedy).toMatch(/detail/i);
expect(result!.remedy).toMatch(/environment|auth|credentials/i);
});
test('classifies error_max_budget_usd as TIMEOUT with specific title', () => {
const result = classifyError(
"Task did not succeed (agent_status='error_max_budget_usd', build_ok=False)",
);
expect(result!.category).toBe(ErrorCategory.TIMEOUT);
expect(result!.title).toBe('Exceeded max budget');
expect(result!.retryable).toBe(true);
expect(result!.remedy).toMatch(/--max-budget/);
});
test('classifies error_during_execution with a mid-turn-error title', () => {
const result = classifyError(
"Task did not succeed (agent_status='error_during_execution', build_ok=False)",
);
expect(result!.category).toBe(ErrorCategory.AGENT);
expect(result!.title).toBe('Agent errored during execution');
expect(result!.retryable).toBe(true);
});
test('classifies the runner.py "Agent session error (subtype=...)" wrapper, not just agent_status= (K5, live-caught ABCA-483)', () => {
// runner.py:515 emits ``Agent session error (subtype='error_max_turns')``
// — a DIFFERENT wrapper from pipeline.py's ``agent_status=``. Pre-K5 this
// fell through to UNKNOWN → "Unexpected error" even though the task hit the
// 100-turn cap (live: a 1-line README task burned 101 turns, reply said
// "Unexpected error"). The pattern must match the subtype= wrapper too.
const turns = classifyError("Agent session error (subtype='error_max_turns')");
expect(turns!.title).toBe('Exceeded max turns');
expect(turns!.category).toBe(ErrorCategory.TIMEOUT);
const budget = classifyError("Agent session error (subtype='error_max_budget_usd')");
expect(budget!.title).toBe('Exceeded max budget');
const exec = classifyError("Agent session error (subtype='error_during_execution')");
expect(exec!.title).toBe('Agent errored during execution');
});
test('matches agent_status with or without quotes around the literal', () => {
// Defensive: the agent writer currently emits single-quoted
// repr values (``agent_status='error_max_turns'``) but a future
// refactor could drop the quotes. The pattern must match both.
const quoted = classifyError("Task did not succeed (agent_status='error_max_turns', build_ok=False)");
const unquoted = classifyError('Task did not succeed (agent_status=error_max_turns, build_ok=False)');
expect(quoted!.title).toBe('Exceeded max turns');
expect(unquoted!.title).toBe('Exceeded max turns');
});
test('classifies receive_response failure', () => {
const result = classifyError(
'receive_response() failed: Connection reset by peer',
);
expect(result!.category).toBe(ErrorCategory.AGENT);
expect(result!.title).toBe('Agent communication failure');
expect(result!.retryable).toBe(true);
});
});
// --- Guardrail errors ---
describe('guardrail errors', () => {
test('classifies guardrail blocked during hydration', () => {
const result = classifyError(
'Hydration failed: Error: Guardrail blocked: CONTENT_POLICY_VIOLATION',
);
expect(result!.category).toBe(ErrorCategory.GUARDRAIL);
expect(result!.title).toBe('Content blocked by guardrail');
expect(result!.retryable).toBe(false);
});
test('classifies direct guardrail blocked message', () => {
const result = classifyError('Guardrail blocked: prompt injection detected');
expect(result!.category).toBe(ErrorCategory.GUARDRAIL);
expect(result!.retryable).toBe(false);
});
test('classifies content policy at submission', () => {
const result = classifyError('Task description was blocked by content policy.');
expect(result!.category).toBe(ErrorCategory.GUARDRAIL);
expect(result!.title).toBe('Content policy violation');
expect(result!.retryable).toBe(false);
});
});
// --- Config errors ---
describe('config errors', () => {
test('classifies Bedrock model not available on deployment', () => {
const result = classifyError(
'The model us.anthropic.claude-sonnet-4-6 is not available on your bedrock deployment. Try --model to switch',
);
expect(result!.category).toBe(ErrorCategory.CONFIG);
expect(result!.title).toBe('Bedrock model not available in this account or Region');
expect(result!.retryable).toBe(false);
});
test('classifies blueprint config load failure', () => {
const result = classifyError(
'Blueprint config load failed: ResourceNotFoundException: Requested resource not found',
);
expect(result!.category).toBe(ErrorCategory.CONFIG);
expect(result!.title).toBe('Blueprint configuration error');
expect(result!.retryable).toBe(true);
});
test('classifies hydration failure (non-guardrail)', () => {
const result = classifyError(
'Hydration failed: Error: Failed to fetch issue body',
);
expect(result!.category).toBe(ErrorCategory.CONFIG);
expect(result!.title).toBe('Context hydration failed');
expect(result!.retryable).toBe(true);
});
test('does not classify hydration + guardrail as config', () => {
const result = classifyError('Hydration failed: Error: Guardrail blocked: xyz');
expect(result!.category).toBe(ErrorCategory.GUARDRAIL);
});
});
// --- Timeout errors ---
describe('timeout errors', () => {
test('classifies orchestrator poll timeout', () => {
const result = classifyError('Orchestrator poll timeout exceeded');
expect(result!.category).toBe(ErrorCategory.TIMEOUT);
expect(result!.title).toBe('Task timed out');
expect(result!.retryable).toBe(false);
});
test('classifies a build/verify command TIMEOUT distinctly from a crash (ABCA-667 live-caught)', () => {
// The fork's full `mise run build` exceeded the 600s cap → Python
// TimeoutExpired. Before this pattern it fell to "Unexpected error"; now it
// reads as a build-time-out (user-actionable: retry / raise the cap), not a
// mysterious crash.
const result = classifyError(
"TimeoutExpired: Command '['bash', '-lc', 'mise run install && MISE_EXPERIMENTAL=1 mise run build']' timed out after 600 seconds",
);
expect(result!.category).toBe(ErrorCategory.TIMEOUT);
expect(result!.title).toMatch(/didn't finish in time|timed out/i);
// A timeout is user-actionable (retry / raise the cap), not a hard failure.
expect(result!.retryable).toBe(true);
// Must NOT fall through to the generic Unexpected error.
expect(result!.title).not.toMatch(/Unexpected error/i);
});
});
// --- Environmental blockers (#251) ---
describe('blocker errors (canonical BLOCKED[<kind>] prefix)', () => {
test('classifies missing_secret and extracts the secret name', () => {
const result = classifyError('BLOCKED[missing_secret]: required secret not wired (resource: OPENAI_API_KEY)');
expect(result!.category).toBe(ErrorCategory.BLOCKED);
expect(result!.title).toBe('Blocked: missing secret');
expect(result!.remedy).toContain('OPENAI_API_KEY');
expect(result!.retryable).toBe(false);
});
test('classifies egress_denied and names the host to allowlist', () => {
const result = classifyError('BLOCKED[egress_denied]: connection refused (resource: registry.npmjs.org)');
expect(result!.category).toBe(ErrorCategory.BLOCKED);
expect(result!.title).toBe('Blocked: egress denied');
expect(result!.remedy).toContain('registry.npmjs.org');
expect(result!.retryable).toBe(false);
});
test('classifies dependency_unreachable as retryable', () => {
const result = classifyError('BLOCKED[dependency_unreachable]: pypi timed out (resource: pypi.org)');
expect(result!.category).toBe(ErrorCategory.BLOCKED);
expect(result!.retryable).toBe(true);
});
test('classifies policy_fail_closed distinctly from a hard-deny', () => {
const result = classifyError('BLOCKED[policy_fail_closed]: Cedar engine unavailable');
expect(result!.category).toBe(ErrorCategory.BLOCKED);
expect(result!.title).toBe('Blocked: policy engine fail-closed');
expect(result!.retryable).toBe(false);
});
test('handles a blocker reason without a resource suffix', () => {
const result = classifyError('BLOCKED[missing_secret]: a required secret was not wired');
expect(result!.category).toBe(ErrorCategory.BLOCKED);
expect(result!.title).toBe('Blocked: missing secret');
});
test('classifies auth_failure (runtime credential rejection → scope advice)', () => {
const result = classifyError('BLOCKED[auth_failure]: credential rejected (resource: github.com)');
expect(result!.category).toBe(ErrorCategory.BLOCKED);
expect(result!.title).toBe('Blocked: authentication rejected');
expect(result!.retryable).toBe(false);
expect(result!.remedy).toContain('scopes');
});
test('auth_failure with a Secrets Manager ARN gives IAM remedy, not PAT scopes (#251 review)', () => {
const arn = 'arn:aws:secretsmanager:us-east-1:123456789012:secret:gh-token-abc';
const result = classifyError(`BLOCKED[auth_failure]: the required GitHub token secret could not be read (resource: ${arn})`);
expect(result!.category).toBe(ErrorCategory.BLOCKED);
expect(result!.title).toBe('Blocked: authentication rejected');
expect(result!.retryable).toBe(false);
// IAM/blueprint advice — NOT the "verify PAT scopes" copy.
expect(result!.remedy).toContain('secretsmanager:GetSecretValue');
expect(result!.remedy).not.toContain('scopes');
});
test('falls back to environmental for an unknown kind', () => {
const result = classifyError('BLOCKED[unknown_environmental]: something odd happened');
expect(result!.category).toBe(ErrorCategory.BLOCKED);
expect(result!.title).toBe('Blocked: environmental fault');
});
test('classifies a BLOCKED prefix appearing mid-message (agent carry-path)', () => {
// failTask persists TaskResult.error verbatim; it may be wrapped.
const result = classifyError('Task failed: BLOCKED[egress_denied]: refused (resource: api.example.com)');
expect(result!.category).toBe(ErrorCategory.BLOCKED);
expect(result!.remedy).toContain('api.example.com');
});
test('extracts resource when the reason is wrapped with trailing text', () => {
// The reason is NOT the end of the string — a wrapper may append context
// or a stack trace after it. Resource extraction must still succeed.
const result = classifyError('BLOCKED[egress_denied]: refused (resource: api.example.com) at step 3');
expect(result!.category).toBe(ErrorCategory.BLOCKED);
expect(result!.remedy).toContain('api.example.com');
});
test('extracts resource when a stack trace follows on a new line', () => {
const result = classifyError(
'BLOCKED[missing_secret]: not wired (resource: OPENAI_API_KEY)\n at foo (bar.py:12)',
);
expect(result!.category).toBe(ErrorCategory.BLOCKED);
expect(result!.remedy).toContain('OPENAI_API_KEY');
});
test('routes a mixed-case kind to the right remedy (case-insensitive)', () => {
const result = classifyError('BLOCKED[Egress_Denied]: refused (resource: host.com)');
expect(result!.category).toBe(ErrorCategory.BLOCKED);
expect(result!.title).toBe('Blocked: egress denied');
});
});
// --- Unknown errors ---
describe('unknown errors', () => {
test('classifies unrecognized error as unknown', () => {
const result = classifyError('Something completely unexpected happened');
expect(result!.category).toBe(ErrorCategory.UNKNOWN);
expect(result!.title).toBe('Unexpected error');
expect(result!.retryable).toBe(false);
});
test('classifies raw Python exception from agent as unknown', () => {
const result = classifyError('ValueError: invalid literal for int()');
expect(result!.category).toBe(ErrorCategory.UNKNOWN);
});
});
// --- Classification shape ---
describe('classification shape', () => {
test('every classification has all required fields', () => {
const messages = [
'Pre-flight check failed: INSUFFICIENT_GITHUB_REPO_PERMISSIONS — Token cannot push to owner/repo',
'Pre-flight check failed: GITHUB_UNREACHABLE — timeout',
'User concurrency limit reached',
'Session start failed: boom',
'Agent SDK stream ended without a ResultMessage (agent_status=unknown).',
'Guardrail blocked: nope',
'Blueprint config load failed: boom',
'The model us.anthropic.claude-sonnet-4-6 is not available on your bedrock deployment.',
'Orchestrator poll timeout exceeded',
'mystery error',
];
for (const msg of messages) {
const result = classifyError(msg) as ErrorClassification;
expect(result).toBeDefined();
expect(typeof result.category).toBe('string');
expect(typeof result.title).toBe('string');
expect(typeof result.description).toBe('string');
expect(typeof result.remedy).toBe('string');
expect(typeof result.retryable).toBe('boolean');
expect(result.title.length).toBeGreaterThan(0);
expect(result.description.length).toBeGreaterThan(0);
expect(result.remedy.length).toBeGreaterThan(0);
// Every classification carries a 3-way errorClass (transient/service/user).
expect([ErrorClass.TRANSIENT, ErrorClass.SERVICE, ErrorClass.USER]).toContain(result.errorClass);
}
});
});
// --- errorClass + retryGuidance (transient vs service vs user) ---
describe('errorClass axis + retryGuidance', () => {
test('the ECS deploy-race is TRANSIENT and isTransientError is true', () => {
const c = classifyError('Session start failed: InvalidParameterException: TaskDefinition is inactive')!;
expect(c.errorClass).toBe(ErrorClass.TRANSIENT);
expect(isTransientError(c)).toBe(true);
});
test('a generic session-start failure is TRANSIENT (compute infra)', () => {
expect(classifyError('Session start failed: boom')!.errorClass).toBe(ErrorClass.TRANSIENT);
});
test('auth/permission is SERVICE (admin fixes it), not transient', () => {
const c = classifyError('INSUFFICIENT_GITHUB_REPO_PERMISSIONS')!;
expect(c.errorClass).toBe(ErrorClass.SERVICE);
expect(isTransientError(c)).toBe(false);
});
test('a build/guardrail failure is USER (change the request/code)', () => {
expect(classifyError('Guardrail blocked: nope')!.errorClass).toBe(ErrorClass.USER);
expect(classifyError('Task did not succeed: agent_status="error_max_turns"')!.errorClass).toBe(ErrorClass.USER);
});
test('retryGuidance: TRANSIENT → "temporary … reply to retry … contact admin if it persists"', () => {
const g = retryGuidance(classifyError('Session start failed: boom')!);
expect(g).toMatch(/temporary infrastructure/i);
expect(g).toMatch(/reply here to try again/i);
expect(g).toMatch(/contact your ABCA admin/i);
});
test('retryGuidance: TRANSIENT + autoRetried → "I automatically tried again and it still failed"', () => {
const g = retryGuidance(classifyError('Session start failed: boom')!, true);
expect(g).toMatch(/automatically tried again/i);
});
test('retryGuidance: SERVICE → "retrying won\'t fix this … your ABCA admin"', () => {
const g = retryGuidance(classifyError('INSUFFICIENT_GITHUB_REPO_PERMISSIONS')!);
expect(g).toMatch(/won'?t fix this/i);
expect(g).toMatch(/admin/i);
expect(g).not.toMatch(/temporary infrastructure/i);
});
test('retryGuidance: USER guardrail → "edit the request"', () => {
const g = retryGuidance(classifyError('Guardrail blocked: nope')!);
expect(g).toMatch(/edit the request/i);
});
// #599 N3: pin the two USER fall-through branches so the #247 failure-renderer
// contract can't rot silently. Built as explicit classifications (the exact
// category/errorClass/retryable each branch keys on) rather than relying on a
// sample string that might reclassify later.
test('retryGuidance: retryable USER (non-guardrail) → "reply here with any extra guidance"', () => {
const cls: ErrorClassification = {
category: ErrorCategory.AGENT,
title: 'build failed',
description: 'the build/test step failed',
remedy: 'fix the failing step',
retryable: true,
errorClass: ErrorClass.USER,
};
const g = retryGuidance(cls);
expect(g).toMatch(/extra guidance/i);
expect(g).toMatch(/try again/i);
expect(g).not.toMatch(/edit the request/i); // not the guardrail branch
});
test('retryGuidance: not-retryable USER/unknown → "a retry may not resolve this"', () => {
const cls: ErrorClassification = {
category: ErrorCategory.UNKNOWN,
title: 'agent reported non-success',
description: 'the agent finished without success',
remedy: 'review the task output',
retryable: false,
errorClass: ErrorClass.USER,
};
const g = retryGuidance(cls);
expect(g).toMatch(/may not resolve this/i);
expect(g).toMatch(/contact your ABCA admin/i);
});
test('isTransientError is false for null / absent classification', () => {
expect(isTransientError(null)).toBe(false);
expect(isTransientError(undefined)).toBe(false);
});
});
// --- Priority / ordering ---
describe('pattern priority', () => {
test('INSUFFICIENT_GITHUB_REPO_PERMISSIONS takes priority over GITHUB_UNREACHABLE substring', () => {
const result = classifyError(
'Pre-flight check failed: INSUFFICIENT_GITHUB_REPO_PERMISSIONS — Token cannot push to owner/repo',
);
expect(result!.category).toBe(ErrorCategory.AUTH);
});
test('guardrail in hydration message takes priority over generic hydration failure', () => {
const result = classifyError('Hydration failed: Error: Guardrail blocked: test');
expect(result!.category).toBe(ErrorCategory.GUARDRAIL);
});
test('agent heartbeat loss matches compute, not agent', () => {
const result = classifyError(
'Agent session lost: no recent heartbeat from the runtime (container may have crashed, been OOM-killed, or stopped)',
);
expect(result!.category).toBe(ErrorCategory.COMPUTE);
});
});
// --- toTaskDetail integration ---
describe('toTaskDetail integration', () => {
const baseRecord: TaskRecord = {
task_id: 'task-1',
user_id: 'user-1',
status: 'FAILED',
repo: 'owner/repo',
resolved_workflow: { id: 'coding/new-task-v1', version: '1.0.0' },
branch_name: 'bgagent/task-1/fix',
channel_source: 'api',
status_created_at: 'FAILED#2026-01-01T00:00:00Z',
created_at: '2026-01-01T00:00:00Z',
updated_at: '2026-01-01T00:00:00Z',
};
test('populates error_classification for a known error pattern', () => {
const record: TaskRecord = { ...baseRecord, error_message: 'User concurrency limit reached' };
const detail = toTaskDetail(record);
expect(detail.error_classification).not.toBeNull();
expect(detail.error_classification!.category).toBe('concurrency');
expect(detail.error_classification!.title).toBe('Concurrency limit reached');
});
test('returns null error_classification when error_message is undefined', () => {
const detail = toTaskDetail(baseRecord);
expect(detail.error_message).toBeNull();
expect(detail.error_classification).toBeNull();
});
test('returns unknown classification for unrecognized error_message', () => {
const record: TaskRecord = { ...baseRecord, error_message: 'ValueError: something broke' };
const detail = toTaskDetail(record);
expect(detail.error_classification).not.toBeNull();
expect(detail.error_classification!.category).toBe('unknown');
});
// Regression: all numeric fields coerce through ``coerceNumericOrNull``
// so the DDB Document-client's string-typed Number deserialization
// cannot leak into downstream consumers (same bug class as the
// ``costUsd.toFixed`` crash fixed in commit ``c09bfd7``). The cast
// to ``unknown as TaskRecord`` simulates a record produced by the
// Document client where ``Number`` attributes came back as strings.
test('coerces string-typed numeric DDB fields to numbers on output', () => {
const record = {
...baseRecord,
duration_s: '12.5',
cost_usd: '0.0042',
max_turns: '30',
max_budget_usd: '1.50',
turns_attempted: '7',
turns_completed: '6',
} as unknown as TaskRecord;
const detail = toTaskDetail(record);
expect(typeof detail.duration_s).toBe('number');
expect(detail.duration_s).toBe(12.5);
expect(typeof detail.cost_usd).toBe('number');
expect(detail.cost_usd).toBe(0.0042);
expect(typeof detail.max_turns).toBe('number');
expect(detail.max_turns).toBe(30);
expect(typeof detail.max_budget_usd).toBe('number');
expect(detail.max_budget_usd).toBe(1.5);
expect(typeof detail.turns_attempted).toBe('number');
expect(detail.turns_attempted).toBe(7);
expect(typeof detail.turns_completed).toBe('number');
expect(detail.turns_completed).toBe(6);
});
test('coerces unparseable numeric strings to null (does not crash)', () => {
const record = {
...baseRecord,
turns_attempted: 'not-a-number',
turns_completed: 'NaN',
} as unknown as TaskRecord;
const detail = toTaskDetail(record);
expect(detail.turns_attempted).toBeNull();
expect(detail.turns_completed).toBeNull();
});
// Compile-time regression for Finding #10 — ``ChannelSource`` is a
// literal union, not ``string``. The ``satisfies`` assertions below
// exercise the valid members; the ``@ts-expect-error`` comments pin
// the narrowing — if someone widens ``ChannelSource`` to ``string``
// these will become un-erroring and fail the build.
test('channel_source narrows to the literal union', () => {
const apiRecord: TaskRecord = { ...baseRecord, channel_source: 'api' };
const webhookRecord: TaskRecord = { ...baseRecord, channel_source: 'webhook' };
const slackRecord: TaskRecord = { ...baseRecord, channel_source: 'slack' };
const linearRecord: TaskRecord = { ...baseRecord, channel_source: 'linear' };
expect(toTaskDetail(apiRecord).channel_source).toBe('api');
expect(toTaskDetail(webhookRecord).channel_source).toBe('webhook');
expect(toTaskDetail(slackRecord).channel_source).toBe('slack');
expect(toTaskDetail(linearRecord).channel_source).toBe('linear');
// @ts-expect-error — 'email' is not a valid ChannelSource
const invalid: TaskRecord = { ...baseRecord, channel_source: 'email' };
// Keep ``invalid`` used so the block doesn't get DCE'd and the
// ``@ts-expect-error`` above remains anchored to a real assignment.
expect(invalid.channel_source).toBeDefined();
});
});
});