Skip to content

Commit dabc1cf

Browse files
jkyberneeesclaude
andcommitted
fix(llm): give SimpleCall the same retry/backoff as the main loop
SimpleCall did a single http.Do with no retry, so any transient 429/5xx or network blip aborted the best-effort secondary features that use it (skill matching, memory summaries, episode extraction, session titles), while the main agent Call retried. Extract the retry loop into postChatWithRetry and route both through it. Test: SimpleCall now succeeds after two 429s (3 attempts). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent aade314 commit dabc1cf

2 files changed

Lines changed: 51 additions & 25 deletions

File tree

internal/llm/client.go

Lines changed: 20 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -209,30 +209,12 @@ func (c *Client) SimpleCall(ctx context.Context, systemPrompt, userPrompt string
209209
return "", fmt.Errorf("llm: marshal request: %w", err)
210210
}
211211

212-
url := c.BaseURL + "/chat/completions"
213-
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(reqBytes))
214-
if err != nil {
215-
return "", fmt.Errorf("llm: create request: %w", err)
216-
}
217-
req.Header.Set("Content-Type", "application/json")
218-
req.Header.Set("Authorization", "Bearer "+c.APIKey)
219-
220-
resp, err := c.http.Do(req)
221-
if err != nil {
222-
return "", fmt.Errorf("llm: %w", err)
223-
}
224-
defer resp.Body.Close()
225-
226-
respBytes, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseSize+1))
212+
// Share the main loop's retry/backoff so a transient blip doesn't abort
213+
// these best-effort secondary calls (skill matching, memory summaries,
214+
// episode extraction, session titles).
215+
respBytes, err := c.postChatWithRetry(ctx, reqBytes)
227216
if err != nil {
228-
return "", fmt.Errorf("llm: read response: %w", err)
229-
}
230-
if len(respBytes) > maxResponseSize {
231-
return "", fmt.Errorf("llm: response exceeds maximum size (%d bytes)", maxResponseSize)
232-
}
233-
234-
if resp.StatusCode != http.StatusOK {
235-
return "", fmt.Errorf("llm: %s (status %d)", resp.Status, resp.StatusCode)
217+
return "", err
236218
}
237219

238220
var raw struct {
@@ -300,14 +282,27 @@ func (c *Client) Call(ctx context.Context, messages []Message, systemBlocks []Sy
300282
return nil, fmt.Errorf("llm: marshal request: %w", err)
301283
}
302284

285+
respBytes, err := c.postChatWithRetry(ctx, reqBytes)
286+
if err != nil {
287+
return nil, err
288+
}
289+
return parseResponse(respBytes)
290+
}
291+
292+
// postChatWithRetry POSTs reqBytes to /chat/completions and returns the raw 200
293+
// response body, retrying transient network errors and retryable HTTP statuses
294+
// (429, 502, 503, 504) with exponential backoff. Shared by every chat call so
295+
// the main loop and the lightweight secondary calls (SimpleCall) get identical
296+
// resilience. Respects ctx cancellation during the backoff sleep.
297+
func (c *Client) postChatWithRetry(ctx context.Context, reqBytes []byte) ([]byte, error) {
303298
url := c.BaseURL + "/chat/completions"
304299

305300
const maxRetries = 3
306301
var lastErr error
307302

308303
for attempt := 0; attempt <= maxRetries; attempt++ {
309304
if attempt > 0 {
310-
// Exponential backoff: 1s, 2s, 4s
305+
// Exponential backoff: 1s, 2s, 4s.
311306
backoff := time.Duration(1<<(attempt-1)) * time.Second
312307
select {
313308
case <-ctx.Done():
@@ -356,7 +351,7 @@ func (c *Client) Call(ctx context.Context, messages []Message, systemBlocks []Sy
356351
return nil, lastErr
357352
}
358353

359-
return parseResponse(respBytes)
354+
return respBytes, nil
360355
}
361356

362357
return nil, fmt.Errorf("llm: retry exhausted (%d attempts): %w", maxRetries+1, lastErr)

internal/llm/retry_test.go

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,37 @@ import (
99
"time"
1010
)
1111

12+
// TestClient_SimpleCall_RetryOn429 verifies the lightweight secondary calls
13+
// share the main loop's retry resilience: a transient 429 no longer aborts a
14+
// skill-match / memory / title call on the first failure.
15+
func TestClient_SimpleCall_RetryOn429(t *testing.T) {
16+
var callCount atomic.Int32
17+
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
18+
count := int(callCount.Add(1))
19+
if count <= 2 {
20+
w.WriteHeader(http.StatusTooManyRequests)
21+
w.Header().Set("Content-Type", "application/json")
22+
w.Write([]byte(`{"error":{"message":"Rate limited"}}`))
23+
return
24+
}
25+
w.Header().Set("Content-Type", "application/json")
26+
w.Write([]byte(`{"choices":[{"message":{"content":"assessed"}}]}`))
27+
}))
28+
defer ts.Close()
29+
30+
c := New(ts.URL, "key", "model", "", 0, 10*time.Second)
31+
out, err := c.SimpleCall(context.Background(), "sys", "user")
32+
if err != nil {
33+
t.Fatalf("unexpected error after retries: %v", err)
34+
}
35+
if out != "assessed" {
36+
t.Errorf("content = %q, want %q", out, "assessed")
37+
}
38+
if callCount.Load() != 3 {
39+
t.Errorf("call count = %d, want 3 (SimpleCall should retry)", callCount.Load())
40+
}
41+
}
42+
1243
func TestClient_Call_RetryOn429(t *testing.T) {
1344
var callCount atomic.Int32
1445
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

0 commit comments

Comments
 (0)