bound default agent sockets

Author: tode-rl

README.md

@@ -340,6 +340,15 @@ This is intended for debugging purposes only and may change in the future withou
 
 By default, this library uses a stable agent for all http/https requests to reuse TCP connections, eliminating many TCP & TLS handshakes and shaving around 100ms off most requests.
 
+The SDK-created default agents cap both active and free sockets at 256 per protocol. Set `httpAgentMaxSockets` to tune that cap for the default agents only:
+
+<!-- prettier-ignore -->
+```ts
+const runloop = new RunloopSDK({
+  httpAgentMaxSockets: 512,
+});
+```
+
 If you would like to disable or customize this behavior, for example to use the API behind a proxy, you can pass an `httpAgent` which is used for all requests (be they http or https), for example:
 
 <!-- prettier-ignore -->
@@ -355,6 +364,8 @@ await runloop.devboxes.create({...}, {
 });
 ```
 
+When `httpAgent` is provided, `httpAgentMaxSockets` is ignored because the custom agent owns its socket policy.
+
 ### HTTP/2 transport
 
 On Node.js, the SDK can send requests over HTTP/2, which multiplexes many concurrent requests over a small number of TLS connections instead of opening a connection per request. Enable it with the `http2` option:

src/_shims/index-deno.ts

@@ -79,7 +79,7 @@ export async function getMultipartRequestOptions<T = Record<string, unknown>>(
   };
 }
 
-export function getDefaultAgent(url: string) {
+export function getDefaultAgent(_url: string, _httpAgentMaxSockets?: number) {
   return undefined;
 }
 export function fileFromPath() {

src/_shims/index.d.ts

@@ -79,7 +79,7 @@ export function getMultipartRequestOptions<T = Record<string, unknown>>(
   opts: RequestOptions<T>,
 ): Promise<RequestOptions<T>>;
 
-export function getDefaultAgent(url: string): any;
+export function getDefaultAgent(url: string, httpAgentMaxSockets?: number): any;
 
 // @ts-ignore
 export type FileFromPathOptions = SelectType<manual.FileFromPathOptions, auto.FileFromPathOptions>;

src/_shims/node-runtime.ts

@@ -19,6 +19,7 @@ import { createH2Fetch } from '../lib/h2-transport';
 type FileFromPathOptions = Omit<FilePropertyBag, 'lastModified'>;
 
 let fileFromPathWarned = false;
+const DEFAULT_HTTP_AGENT_MAX_SOCKETS = 256;
 
 /**
  * @deprecated use fs.createReadStream('./my/file.txt') instead
@@ -39,20 +40,26 @@ async function fileFromPath(path: string, ...args: any[]): Promise<File> {
   return await _fileFromPath(path, ...args);
 }
 
-const defaultHttpAgent: Agent = new KeepAliveAgent({
-  keepAlive: true,
-  timeout: 10 * 60 * 1000,
-  maxSockets: Infinity,
-  maxFreeSockets: 2048,
-  freeSocketTimeout: 30_000,
-});
-const defaultHttpsAgent: Agent = new KeepAliveAgent.HttpsAgent({
-  keepAlive: true,
-  timeout: 10 * 60 * 1000,
-  maxSockets: Infinity,
-  maxFreeSockets: 2048,
-  freeSocketTimeout: 30_000,
-});
+const defaultHttpAgents = new Map<number, Agent>();
+const defaultHttpsAgents = new Map<number, Agent>();
+
+function getOrCreateDefaultAgent(url: string, httpAgentMaxSockets = DEFAULT_HTTP_AGENT_MAX_SOCKETS): Agent {
+  const isHttps = url.startsWith('https');
+  const cache = isHttps ? defaultHttpsAgents : defaultHttpAgents;
+  const cached = cache.get(httpAgentMaxSockets);
+  if (cached) return cached;
+
+  const options = {
+    keepAlive: true,
+    timeout: 10 * 60 * 1000,
+    maxSockets: httpAgentMaxSockets,
+    maxFreeSockets: httpAgentMaxSockets,
+    freeSocketTimeout: 30_000,
+  };
+  const agent: Agent = isHttps ? new KeepAliveAgent.HttpsAgent(options) : new KeepAliveAgent(options);
+  cache.set(httpAgentMaxSockets, agent);
+  return agent;
+}
 
 async function getMultipartRequestOptions<T = Record<string, unknown>>(
   form: fd.FormData,
@@ -88,7 +95,7 @@ export function getRuntime(): Shims {
     File: fd.File,
     ReadableStream,
     getMultipartRequestOptions,
-    getDefaultAgent: (url: string): Agent => (url.startsWith('https') ? defaultHttpsAgent : defaultHttpAgent),
+    getDefaultAgent: getOrCreateDefaultAgent,
     fileFromPath,
     isFsReadStream: (value: any): value is FsReadStream => value instanceof FsReadStream,
   };

src/_shims/registry.ts

@@ -25,7 +25,7 @@ export interface Shims {
     form: Shims['FormData'],
     opts: RequestOptions<T>,
   ) => Promise<RequestOptions<T>>;
-  getDefaultAgent: (url: string) => any;
+  getDefaultAgent: (url: string, httpAgentMaxSockets?: number) => any;
   fileFromPath:
     | ((path: string, filename?: string, options?: {}) => Promise<Shims['File']>)
     | ((path: string, options?: {}) => Promise<Shims['File']>);

src/_shims/web-runtime.ts

@@ -96,7 +96,7 @@ export function getRuntime({ manuallyImported }: { manuallyImported?: boolean }
       ...opts,
       body: new MultipartBody(form) as any,
     }),
-    getDefaultAgent: (url: string) => undefined,
+    getDefaultAgent: (_url: string, _httpAgentMaxSockets?: number) => undefined,
     fileFromPath: () => {
       throw new Error(
         'The `fileFromPath` function is only supported in Node. See the README for more details: https://www.github.com/runloopai/api-client-ts#file-uploads',

src/core.ts

@@ -211,6 +211,7 @@ export abstract class APIClient {
   maxRetries: number;
   timeout: number;
   httpAgent: Agent | undefined;
+  httpAgentMaxSockets: number;
 
   private fetch: Fetch;
   protected idempotencyHeader?: string;
@@ -221,20 +222,23 @@ export abstract class APIClient {
     maxRetries = 5,
     timeout = 30000, // 30 seconds
     httpAgent,
+    httpAgentMaxSockets = 256,
     fetch: overriddenFetch,
   }: {
     baseURL: string;
     baseURLOverridden: boolean;
     maxRetries?: number | undefined;
     timeout: number | undefined;
     httpAgent: Agent | undefined;
+    httpAgentMaxSockets?: number | undefined;
     fetch: Fetch | undefined;
   }) {
     this.baseURL = baseURL;
     this.#baseURLOverridden = baseURLOverridden;
-    this.maxRetries = validatePositiveInteger('maxRetries', maxRetries);
+    this.maxRetries = validateNonNegativeInteger('maxRetries', maxRetries);
     this.timeout = validatePositiveInteger('timeout', timeout);
     this.httpAgent = httpAgent;
+    this.httpAgentMaxSockets = validatePositiveInteger('httpAgentMaxSockets', httpAgentMaxSockets);
 
     this.fetch = overriddenFetch ?? fetch;
   }
@@ -354,7 +358,7 @@ export abstract class APIClient {
     const url = this.buildURL(path!, query, defaultBaseURL);
     if ('timeout' in options) validatePositiveInteger('timeout', options.timeout);
     options.timeout = options.timeout ?? this.timeout;
-    const httpAgent = options.httpAgent ?? this.httpAgent ?? getDefaultAgent(url);
+    const httpAgent = options.httpAgent ?? this.httpAgent ?? getDefaultAgent(url, this.httpAgentMaxSockets);
     const minAgentTimeout = options.timeout + 1000;
     if (
       typeof (httpAgent as any)?.options?.timeout === 'number' &&
@@ -1055,12 +1059,22 @@ const validatePositiveInteger = (name: string, n: unknown): number => {
   if (typeof n !== 'number' || !Number.isInteger(n)) {
     throw new RunloopError(`${name} must be an integer`);
   }
-  if (n < 0) {
+  if (n <= 0) {
     throw new RunloopError(`${name} must be a positive integer`);
   }
   return n;
 };
 
+const validateNonNegativeInteger = (name: string, n: unknown): number => {
+  if (typeof n !== 'number' || !Number.isInteger(n)) {
+    throw new RunloopError(`${name} must be an integer`);
+  }
+  if (n < 0) {
+    throw new RunloopError(`${name} must be a non-negative integer`);
+  }
+  return n;
+};
+
 export const castToError = (err: any): Error => {
   if (err instanceof Error) return err;
   if (typeof err === 'object' && err !== null) {

src/index.ts

@@ -283,6 +283,16 @@ export interface ClientOptions {
    */
   httpAgent?: Agent | undefined;
 
+  /**
+   * Maximum active and free sockets for the SDK-created default Node.js HTTP(S) agents.
+   *
+   * This only applies when `httpAgent` is not provided. Custom `httpAgent` instances own
+   * their full socket policy.
+   *
+   * @default 256
+   */
+  httpAgentMaxSockets?: number | undefined;
+
   /**
    * Specify a custom `fetch` function implementation.
    *
@@ -358,6 +368,7 @@ export class Runloop extends Core.APIClient {
    * @param {string} [opts.baseURL=process.env['RUNLOOP_BASE_URL'] ?? https://api.runloop.ai] - Override the default base URL for the API.
    * @param {number} [opts.timeout=30 seconds] - The maximum amount of time (in milliseconds) the client will wait for a response before timing out.
    * @param {number} [opts.httpAgent] - An HTTP agent used to manage HTTP(s) connections.
+   * @param {number} [opts.httpAgentMaxSockets=256] - Maximum sockets for SDK-created default HTTP(S) agents.
    * @param {Core.Fetch} [opts.fetch] - Specify a custom `fetch` function implementation.
    * @param {boolean | H2FetchOptions} [opts.http2=false] - Send requests over HTTP/2 (Node only; ignored when `fetch` is provided). `true` uses the default bounded pool; pass H2FetchOptions to tune.
    * @param {number} [opts.maxRetries=5] - The maximum number of times the client will retry a request.
@@ -397,6 +408,7 @@ export class Runloop extends Core.APIClient {
       baseURLOverridden: baseURL ? baseURL !== 'https://api.runloop.ai' : false,
       timeout: options.timeout ?? 30000 /* 30 seconds */,
       httpAgent: options.httpAgent,
+      httpAgentMaxSockets: options.httpAgentMaxSockets,
       maxRetries: options.maxRetries,
       fetch:
         options.fetch ??

tests/index.test.ts

@@ -151,6 +151,60 @@ describe('instantiate client', () => {
     }
   });
 
+  describe('httpAgentMaxSockets', () => {
+    test('bounds SDK-created default agents to 256 sockets', async () => {
+      const client = new Runloop({
+        baseURL: 'http://localhost:5000/',
+        bearerToken: 'My Bearer Token',
+      });
+
+      const { req } = await client.buildRequest({ path: '/foo', method: 'get' });
+      const agent = (req as any).agent;
+
+      expect(agent.maxSockets).toEqual(256);
+      expect(agent.maxFreeSockets).toEqual(256);
+    });
+
+    test('uses httpAgentMaxSockets for SDK-created default agents', async () => {
+      const client = new Runloop({
+        baseURL: 'https://localhost:5000/',
+        bearerToken: 'My Bearer Token',
+        httpAgentMaxSockets: 512,
+      });
+
+      const { req } = await client.buildRequest({ path: '/foo', method: 'get' });
+      const agent = (req as any).agent;
+
+      expect(agent.maxSockets).toEqual(512);
+      expect(agent.maxFreeSockets).toEqual(512);
+    });
+
+    test('custom httpAgent wins over httpAgentMaxSockets', async () => {
+      const httpAgent = { options: { timeout: 10 * 60 * 1000 } } as any;
+      const client = new Runloop({
+        baseURL: 'http://localhost:5000/',
+        bearerToken: 'My Bearer Token',
+        httpAgent,
+        httpAgentMaxSockets: 512,
+      });
+
+      const { req } = await client.buildRequest({ path: '/foo', method: 'get' });
+
+      expect((req as any).agent).toBe(httpAgent);
+    });
+
+    test('rejects non-positive httpAgentMaxSockets', () => {
+      expect(
+        () =>
+          new Runloop({
+            baseURL: 'http://localhost:5000/',
+            bearerToken: 'My Bearer Token',
+            httpAgentMaxSockets: 0,
+          }),
+      ).toThrow('httpAgentMaxSockets must be a positive integer');
+    });
+  });
+
   test('explicit global fetch', async () => {
     // make sure the global fetch type is assignable to our Fetch type
     const client = new Runloop({