Skip to content

Commit 2e5b2ed

Browse files
committed
perf: replace undici fetch with custom https.Agent for long keepAlive
The default undici-based global fetch only keeps connections alive for 4 seconds, which is too short for a CLI where the user may spend 10–30 seconds reading output before typing the next prompt. Add a custom fetch implementation backed by node:https.Agent with keepAlive: true and a 60-second idle timeout. The custom fetch is passed to the OpenAI SDK constructor so every LLM API request benefits from persistent connections across conversational turns. Also handles streaming request bodies (ReadableStream) for SDK features like file uploads.
1 parent 1bd7e6a commit 2e5b2ed

1 file changed

Lines changed: 95 additions & 0 deletions

File tree

src/ui/App.tsx

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import React, { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useSta
22
import { Box, Static, Text, useApp, useStdout, useWindowSize } from "ink";
33
import chalk from "chalk";
44
import * as fs from "fs";
5+
import https from "node:https";
56
import * as os from "os";
67
import * as path from "path";
78
import OpenAI from "openai";
@@ -728,6 +729,99 @@ export function resolveCurrentSettings(projectRoot: string = process.cwd()): Res
728729
);
729730
}
730731

732+
// Custom fetch implementation that uses node:https.Agent with a configurable
733+
// keepAlive timeout. The default undici-based global fetch only keeps
734+
// connections alive for 4 seconds, which is too short for a CLI where the
735+
// user may spend 10–30 seconds reading output before typing the next prompt.
736+
// With this custom Agent we get full control over idle connection lifetime.
737+
const KEEP_ALIVE_MSEC = 60_000; // 1 minute
738+
739+
function createCustomFetch(keepAliveMsecs: number = KEEP_ALIVE_MSEC) {
740+
const agent = new https.Agent({ keepAlive: true, keepAliveMsecs });
741+
742+
return async function customFetch(url: string | URL | Request, init?: RequestInit): Promise<Response> {
743+
const urlObj = typeof url === "string" ? new URL(url) : url instanceof URL ? url : new URL(url.url);
744+
const { method = "GET", headers = {}, body: reqBody, signal } = init ?? {};
745+
746+
// Normalize Headers to a plain Record
747+
const plainHeaders: Record<string, string> = {};
748+
if (headers instanceof Headers) {
749+
for (const [k, v] of headers) plainHeaders[k] = v;
750+
} else if (Array.isArray(headers)) {
751+
for (const [k, v] of headers) plainHeaders[k] = v;
752+
} else {
753+
Object.assign(plainHeaders, headers);
754+
}
755+
756+
const port = urlObj.port ? Number(urlObj.port) : 443;
757+
758+
return new Promise((resolve, reject) => {
759+
const req = https.request(
760+
{
761+
hostname: urlObj.hostname,
762+
port,
763+
path: urlObj.pathname + urlObj.search,
764+
method,
765+
headers: plainHeaders,
766+
agent,
767+
signal: signal ?? undefined,
768+
},
769+
(res) => {
770+
const resHeaders = new Headers();
771+
for (const [k, v] of Object.entries(res.headers)) {
772+
if (v) (Array.isArray(v) ? v : [v]).forEach((val) => resHeaders.append(k, val));
773+
}
774+
775+
const body = new ReadableStream({
776+
start(controller) {
777+
res.on("data", (chunk: Buffer) => controller.enqueue(new Uint8Array(chunk)));
778+
res.on("end", () => controller.close());
779+
res.on("error", (err) => controller.error(err));
780+
},
781+
cancel() {
782+
res.destroy();
783+
},
784+
});
785+
786+
resolve(
787+
new Response(body, {
788+
status: res.statusCode,
789+
statusText: res.statusMessage,
790+
headers: resHeaders,
791+
})
792+
);
793+
}
794+
);
795+
796+
req.on("error", reject);
797+
798+
if (reqBody) {
799+
if (typeof reqBody === "string" || reqBody instanceof Uint8Array || ArrayBuffer.isView(reqBody)) {
800+
req.write(reqBody as Parameters<typeof req.write>[0]);
801+
} else if (reqBody instanceof ReadableStream) {
802+
// Pipe streaming request body (used for file uploads by the SDK)
803+
const reader = (reqBody as ReadableStream<Uint8Array>).getReader();
804+
(async () => {
805+
try {
806+
while (true) {
807+
const { done, value } = await reader.read();
808+
if (done) break;
809+
if (value) req.write(value);
810+
}
811+
req.end();
812+
} catch (err) {
813+
req.destroy(err instanceof Error ? err : new Error(String(err)));
814+
}
815+
})();
816+
return; // req.end() is called inside the async IIFE
817+
}
818+
}
819+
820+
req.end();
821+
});
822+
};
823+
}
824+
731825
// Module-level cache for the OpenAI client instance. The client itself is
732826
// a stateless fetch wrapper, so it is safe to share across calls as long as
733827
// the apiKey + baseURL stay the same. Model, thinking-mode and other
@@ -782,6 +876,7 @@ export function createOpenAIClient(projectRoot: string = process.cwd()): {
782876
_cachedOpenAI = new OpenAI({
783877
apiKey: settings.apiKey,
784878
baseURL: settings.baseURL || undefined,
879+
fetch: createCustomFetch(),
785880
});
786881
_cachedOpenAIKey = cacheKey;
787882

0 commit comments

Comments
 (0)