Fix exponential backoff for provider: {anthropic,openai}

Exponential backoff was broken for anthropic. The reason is that ParseError()
returned a plain Error with no .response property. withExponentialBackoff() only
triggers the backoff logic when when error.response?.status == 429. Without that
property, it immediately rethrows the error. With this fix, it correctly backs
off on an HTTP 429 error.

`provider: openai` bypasses BaseLLM.fetch() and uses the openai SDK.

Fix the tests so that they pass again.

Some APIs have a per minute rate-limt, so increase the max retries in both
codepaths to 8 so that it it retries up to 127.5s until it fails.

Author: leegarrett

core/llm/index.ts

@@ -460,9 +460,11 @@ export abstract class BaseLLM implements ILLM {
         );
       }
     }
-    return new Error(
+    const error = new Error(
       `HTTP ${resp.status} ${resp.statusText} from ${resp.url}\n\n${text}`,
-    );
+    ) as any;
+    error.response = resp;
+    return error;
   }
 
   fetch(url: RequestInfo | URL, init?: RequestInit): Promise<Response> {
@@ -543,7 +545,7 @@ export abstract class BaseLLM implements ILLM {
     };
     return withExponentialBackoff<Response>(
       () => customFetch(url, init) as any,
-      5,
+      8,
       0.5,
     );
   }

core/util/withExponentialBackoff.test.ts

@@ -6,7 +6,7 @@ import {
   RETRY_AFTER_HEADER,
 } from "./withExponentialBackoff";
 
-describe.skip("withExponentialBackoff", () => {
+describe("withExponentialBackoff", () => {
   it("should return result when apiCall succeeds on first attempt", async () => {
     // Arrange
     const apiCall = jest.fn().mockResolvedValue("Success");
@@ -112,7 +112,7 @@ describe.skip("withExponentialBackoff", () => {
     // Act & Assert
     await expect(
       withExponentialBackoff(apiCall, maxTries, initialDelaySeconds),
-    ).rejects.toThrow("Failed to make API call after max tries");
+    ).rejects.toThrow(`Failed to make API call after ${maxTries} retries`);
 
     expect(apiCall).toHaveBeenCalledTimes(maxTries);
   });
@@ -135,7 +135,7 @@ describe.skip("withExponentialBackoff", () => {
 
     await expect(
       withExponentialBackoff(apiCall, maxTries, initialDelaySeconds),
-    ).rejects.toThrow("Failed to make API call after max tries");
+    ).rejects.toThrow(`Failed to make API call after ${maxTries} retries`);
 
     expect(apiCall).toHaveBeenCalledTimes(0);
   });

packages/openai-adapters/src/apis/OpenAI.ts

@@ -46,6 +46,7 @@ export class OpenAIApi implements BaseLlmApi {
       baseURL: this.apiBase,
       fetch: customFetch(config.requestOptions),
       timeout: config?.requestOptions?.timeout || undefined,
+      maxRetries: 8,
     });
   }
   modifyChatBody<T extends ChatCompletionCreateParams>(body: T): T {