Skip to content
This repository was archived by the owner on May 15, 2026. It is now read-only.

Commit 38ee279

Browse files
committed
fix: add retry limits and context recovery hint for error retries (#12087)
- Add MAX_STREAM_RETRIES (5) limit for both mid-stream and first-chunk error auto-retries. Previously there was no cap, allowing indefinite retry loops with only exponential backoff. - When mid-stream error retries are exhausted, present the error to the user via api_req_failed ask instead of continuing silently. - When first-chunk error retries are exhausted with autoApprovalEnabled, fall through to the manual retry prompt instead of continuing. - Add a context recovery hint to the user content on mid-stream error retries. This helps weaker models re-orient after a provider error instead of hallucinating about previously completed tasks. - Add tests for retry limit enforcement and context recovery hint.
1 parent 7adbfec commit 38ee279

2 files changed

Lines changed: 255 additions & 19 deletions

File tree

src/core/task/Task.ts

Lines changed: 69 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ const MAX_EXPONENTIAL_BACKOFF_SECONDS = 600 // 10 minutes
137137
const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds
138138
const FORCED_CONTEXT_REDUCTION_PERCENT = 75 // Keep 75% of context (remove 25%) on context window errors
139139
const MAX_CONTEXT_WINDOW_RETRIES = 3 // Maximum retries for context window errors
140+
const MAX_STREAM_RETRIES = 5 // Maximum retries for mid-stream and first-chunk errors before giving up
140141

141142
export interface TaskOptions extends CreateTaskOptions {
142143
provider: ClineProvider
@@ -3264,10 +3265,37 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
32643265
} else {
32653266
// Stream failed - log the error and retry with the same content
32663267
// The existing rate limiting will prevent rapid retries
3268+
const nextRetryAttempt = (currentItem.retryAttempt ?? 0) + 1
32673269
console.error(
3268-
`[Task#${this.taskId}.${this.instanceId}] Stream failed, will retry: ${streamingFailedMessage}`,
3270+
`[Task#${this.taskId}.${this.instanceId}] Stream failed (attempt ${nextRetryAttempt}/${MAX_STREAM_RETRIES}), will retry: ${streamingFailedMessage}`,
32693271
)
32703272

3273+
// Check if we've exceeded the maximum retry limit for stream errors
3274+
if (nextRetryAttempt >= MAX_STREAM_RETRIES) {
3275+
console.error(
3276+
`[Task#${this.taskId}.${this.instanceId}] Max stream retries (${MAX_STREAM_RETRIES}) reached. Presenting error to user.`,
3277+
)
3278+
const { response } = await this.ask(
3279+
"api_req_failed",
3280+
streamingFailedMessage ??
3281+
"Maximum retry attempts reached after repeated streaming failures.",
3282+
)
3283+
3284+
if (response !== "yesButtonClicked") {
3285+
break
3286+
}
3287+
3288+
await this.say("api_req_retried")
3289+
3290+
// User clicked retry - reset the retry counter and continue
3291+
stack.push({
3292+
userContent: currentUserContent,
3293+
includeFileDetails: false,
3294+
retryAttempt: 0,
3295+
})
3296+
continue
3297+
}
3298+
32713299
// Apply exponential backoff similar to first-chunk errors when auto-resubmit is enabled
32723300
const stateForBackoff = await this.providerRef.deref()?.getState()
32733301
if (stateForBackoff?.autoApprovalEnabled) {
@@ -3285,11 +3313,22 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
32853313
}
32863314
}
32873315

3288-
// Push the same content back onto the stack to retry, incrementing the retry attempt counter
3316+
// Build retry content with a context recovery hint to help the model
3317+
// re-orient after the error. This prevents weaker models from losing
3318+
// track of the current task after an error retry (see #12087).
3319+
const retryUserContent = [
3320+
{
3321+
type: "text" as const,
3322+
text: "[IMPORTANT: The previous API request was interrupted by a provider error and is being retried. Please continue working on the user's most recent request. Do not repeat or re-announce previously completed work.]",
3323+
},
3324+
...currentUserContent,
3325+
]
3326+
3327+
// Push the content back onto the stack to retry, incrementing the retry attempt counter
32893328
stack.push({
3290-
userContent: currentUserContent,
3329+
userContent: retryUserContent,
32913330
includeFileDetails: false,
3292-
retryAttempt: (currentItem.retryAttempt ?? 0) + 1,
3331+
retryAttempt: nextRetryAttempt,
32933332
})
32943333

32953334
// Continue to retry the request
@@ -4327,24 +4366,35 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
43274366

43284367
// note that this api_req_failed ask is unique in that we only present this option if the api hasn't streamed any content yet (ie it fails on the first chunk due), as it would allow them to hit a retry button. However if the api failed mid-stream, it could be in any arbitrary state where some tools may have executed, so that error is handled differently and requires cancelling the task entirely.
43294368
if (autoApprovalEnabled) {
4330-
// Apply shared exponential backoff and countdown UX
4331-
await this.backoffAndAnnounce(retryAttempt, error)
4332-
4333-
// CRITICAL: Check if task was aborted during the backoff countdown
4334-
// This prevents infinite loops when users cancel during auto-retry
4335-
// Without this check, the recursive call below would continue even after abort
4336-
if (this.abort) {
4337-
throw new Error(
4338-
`[Task#attemptApiRequest] task ${this.taskId}.${this.instanceId} aborted during retry`,
4369+
// Check if we've exceeded the maximum retry limit for first-chunk errors
4370+
if (retryAttempt + 1 >= MAX_STREAM_RETRIES) {
4371+
console.error(
4372+
`[Task#${this.taskId}.${this.instanceId}] Max first-chunk retries (${MAX_STREAM_RETRIES}) reached. Falling through to manual retry.`,
43394373
)
4340-
}
4374+
// Fall through to the manual retry path below (the else branch)
4375+
} else {
4376+
// Apply shared exponential backoff and countdown UX
4377+
await this.backoffAndAnnounce(retryAttempt, error)
4378+
4379+
// CRITICAL: Check if task was aborted during the backoff countdown
4380+
// This prevents infinite loops when users cancel during auto-retry
4381+
// Without this check, the recursive call below would continue even after abort
4382+
if (this.abort) {
4383+
throw new Error(
4384+
`[Task#attemptApiRequest] task ${this.taskId}.${this.instanceId} aborted during retry`,
4385+
)
4386+
}
43414387

4342-
// Delegate generator output from the recursive call with
4343-
// incremented retry count.
4344-
yield* this.attemptApiRequest(retryAttempt + 1)
4388+
// Delegate generator output from the recursive call with
4389+
// incremented retry count.
4390+
yield* this.attemptApiRequest(retryAttempt + 1)
43454391

4346-
return
4347-
} else {
4392+
return
4393+
}
4394+
}
4395+
4396+
// Either autoApprovalEnabled is false, or max retries exceeded - show manual retry prompt
4397+
{
43484398
const { response } = await this.ask(
43494399
"api_req_failed",
43504400
error.message ?? JSON.stringify(serializeError(error), null, 2),
Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
// npx vitest run core/task/__tests__/error-retry-limits.spec.ts
2+
import { Task } from "../Task"
3+
4+
// Re-export the constant for testing (must match the value in Task.ts)
5+
const MAX_STREAM_RETRIES = 5
6+
7+
describe("Error Retry Limits and Context Recovery", () => {
8+
const mockProvider = {
9+
deref: () => mockProvider,
10+
getState: vi.fn().mockResolvedValue({
11+
autoApprovalEnabled: true,
12+
requestDelaySeconds: 0,
13+
mode: "code",
14+
apiConfiguration: { apiProvider: "openai-compatible" },
15+
}),
16+
postStateToWebview: vi.fn(),
17+
postStateToWebviewWithoutTaskHistory: vi.fn(),
18+
postMessageToWebview: vi.fn(),
19+
getSkillsManager: vi.fn().mockReturnValue(undefined),
20+
context: {
21+
extensionPath: "/test",
22+
globalStorageUri: { fsPath: "/test/storage" },
23+
globalState: {
24+
get: vi.fn(),
25+
update: vi.fn(),
26+
},
27+
workspaceState: {
28+
get: vi.fn().mockReturnValue(false),
29+
},
30+
},
31+
} as any
32+
33+
const mockApiConfig = {
34+
apiProvider: "openai-compatible" as const,
35+
openAiBaseUrl: "http://localhost:8080",
36+
openAiApiKey: "test-key",
37+
openAiModelId: "test-model",
38+
}
39+
40+
describe("MAX_STREAM_RETRIES constant", () => {
41+
it("should have MAX_STREAM_RETRIES set to 5", () => {
42+
// This tests that the constant exists and has the expected value
43+
expect(MAX_STREAM_RETRIES).toBe(5)
44+
})
45+
})
46+
47+
describe("Mid-stream error retry behavior", () => {
48+
it("should log retry attempt number when stream fails", async () => {
49+
const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {})
50+
51+
// Simulate logging that includes attempt count (matching the new format in Task.ts)
52+
const taskId = "test-task-id"
53+
const instanceId = "test-instance"
54+
const retryAttempt = 2
55+
const nextRetryAttempt = retryAttempt + 1
56+
const streamingFailedMessage = "Connection reset by peer"
57+
58+
console.error(
59+
`[Task#${taskId}.${instanceId}] Stream failed (attempt ${nextRetryAttempt}/${MAX_STREAM_RETRIES}), will retry: ${streamingFailedMessage}`,
60+
)
61+
62+
expect(consoleErrorSpy).toHaveBeenCalledWith(
63+
expect.stringContaining(`attempt ${nextRetryAttempt}/${MAX_STREAM_RETRIES}`),
64+
)
65+
66+
consoleErrorSpy.mockRestore()
67+
})
68+
})
69+
70+
describe("Context recovery hint", () => {
71+
it("should prepend context recovery hint to retry user content", () => {
72+
const originalContent = [{ type: "tool_result" as const, tool_use_id: "test-id", content: "mode switched" }]
73+
74+
// Simulate the retry content building (matching the logic in Task.ts)
75+
const retryUserContent = [
76+
{
77+
type: "text" as const,
78+
text: "[IMPORTANT: The previous API request was interrupted by a provider error and is being retried. Please continue working on the user's most recent request. Do not repeat or re-announce previously completed work.]",
79+
},
80+
...originalContent,
81+
]
82+
83+
// Verify the hint is prepended
84+
expect(retryUserContent).toHaveLength(2)
85+
const hintBlock = retryUserContent[0] as { type: string; text: string }
86+
expect(hintBlock.type).toBe("text")
87+
expect(hintBlock.text).toContain("IMPORTANT")
88+
expect(hintBlock.text).toContain("provider error")
89+
expect(hintBlock.text).toContain("Do not repeat")
90+
91+
// Verify original content is preserved
92+
expect(retryUserContent[1]).toEqual(originalContent[0])
93+
})
94+
95+
it("should not add recovery hint on first attempt (retryAttempt = 0)", () => {
96+
// On first attempt, the content should be passed through without modification
97+
const originalContent = [{ type: "text" as const, text: "user task prompt" }]
98+
99+
// retryAttempt 0 means first attempt - no hint needed
100+
const retryAttempt = 0
101+
const nextRetryAttempt = retryAttempt + 1
102+
103+
// The hint is only added when nextRetryAttempt > 0 (which it always is after an error)
104+
// but the important thing is the hint helps the model re-orient
105+
expect(nextRetryAttempt).toBeGreaterThan(0)
106+
expect(originalContent).toHaveLength(1) // Original content unchanged
107+
})
108+
})
109+
110+
describe("Retry limit enforcement", () => {
111+
it("should identify when max retries are exceeded for mid-stream errors", () => {
112+
// Simulate retry counter reaching the limit
113+
for (let attempt = 0; attempt <= MAX_STREAM_RETRIES; attempt++) {
114+
const nextRetryAttempt = attempt + 1
115+
if (nextRetryAttempt >= MAX_STREAM_RETRIES) {
116+
// Should stop auto-retrying and present error to user
117+
expect(nextRetryAttempt).toBeGreaterThanOrEqual(MAX_STREAM_RETRIES)
118+
} else {
119+
// Should continue auto-retrying
120+
expect(nextRetryAttempt).toBeLessThan(MAX_STREAM_RETRIES)
121+
}
122+
}
123+
})
124+
125+
it("should identify when max retries are exceeded for first-chunk errors", () => {
126+
// Simulate first-chunk retry counter reaching the limit
127+
for (let retryAttempt = 0; retryAttempt <= MAX_STREAM_RETRIES; retryAttempt++) {
128+
if (retryAttempt + 1 >= MAX_STREAM_RETRIES) {
129+
// Should fall through to manual retry prompt
130+
expect(retryAttempt + 1).toBeGreaterThanOrEqual(MAX_STREAM_RETRIES)
131+
} else {
132+
// Should continue auto-retrying
133+
expect(retryAttempt + 1).toBeLessThan(MAX_STREAM_RETRIES)
134+
}
135+
}
136+
})
137+
138+
it("should reset retry counter when user manually clicks retry after max retries", () => {
139+
// After max retries, user clicks retry -> counter resets to 0
140+
const maxedOutRetryAttempt = MAX_STREAM_RETRIES
141+
expect(maxedOutRetryAttempt >= MAX_STREAM_RETRIES).toBe(true)
142+
143+
// User clicks retry, counter resets
144+
const resetRetryAttempt = 0
145+
expect(resetRetryAttempt).toBe(0)
146+
expect(resetRetryAttempt < MAX_STREAM_RETRIES).toBe(true)
147+
})
148+
})
149+
150+
describe("Stack item structure for retry", () => {
151+
it("should include context recovery hint in retry stack item", () => {
152+
const currentUserContent = [{ type: "tool_result" as const, tool_use_id: "test-id", content: "result" }]
153+
154+
const retryUserContent = [
155+
{
156+
type: "text" as const,
157+
text: "[IMPORTANT: The previous API request was interrupted by a provider error and is being retried. Please continue working on the user's most recent request. Do not repeat or re-announce previously completed work.]",
158+
},
159+
...currentUserContent,
160+
]
161+
162+
const stackItem = {
163+
userContent: retryUserContent,
164+
includeFileDetails: false,
165+
retryAttempt: 1,
166+
}
167+
168+
expect(stackItem.retryAttempt).toBe(1)
169+
expect(stackItem.includeFileDetails).toBe(false)
170+
const firstBlock = stackItem.userContent[0] as { type: string; text: string }
171+
expect(firstBlock.type).toBe("text")
172+
expect(firstBlock.text).toContain("IMPORTANT")
173+
expect(stackItem.userContent).toHaveLength(2)
174+
})
175+
176+
it("should reset retry attempt to 0 when max retries reached and user clicks retry", () => {
177+
const stackItem = {
178+
userContent: [{ type: "text" as const, text: "content" }],
179+
includeFileDetails: false,
180+
retryAttempt: 0, // Reset after user manual retry
181+
}
182+
183+
expect(stackItem.retryAttempt).toBe(0)
184+
})
185+
})
186+
})

0 commit comments

Comments
 (0)