Skip to content

Commit 04486db

Browse files
JeevantheDevclaude
andcommitted
perf: eliminate double LLM call and reduce GitHub API timeout in AI chat
- Reuse non-streaming probe response as text stream instead of making a second streaming LLM call when no tool use is detected, cutting chat latency roughly in half for plain Q&A responses - Reduce GitHub commit data timeout from 5s to 2s since it is best-effort context that should not block the AI response on slow API calls Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 1e79793 commit 04486db

2 files changed

Lines changed: 8 additions & 10 deletions

File tree

lib/agent/context.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,9 @@ export async function getShipBrainAgentContext({
115115
{ includeGitHubData: true } // #1: Enable GitHub commits
116116
);
117117

118-
// Race against a 5-second timeout — commits are best-effort
118+
// Race against a 2-second timeout — commits are best-effort
119119
const timeoutPromise = new Promise<null>((resolve) =>
120-
setTimeout(() => resolve(null), 5000)
120+
setTimeout(() => resolve(null), 2000)
121121
);
122122

123123
deploymentContext = await Promise.race([deploymentContextPromise, timeoutPromise]);

lib/ai/shipbrain-chat.ts

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1260,10 +1260,6 @@ export async function streamShipBrainQuestion(input: {
12601260
return { ...base, action, stream: textStream(confirmMsg) };
12611261
}
12621262

1263-
// Build model with tools and stream
1264-
const model = getModel({ temperature: 0.2, streaming: true });
1265-
const modelWithTools = model.bind({ tools: getLangChainToolSpecs() } as any);
1266-
12671263
// #8 & #9: Build a richer context preamble with memory notes and notification counts
12681264
const memoryBlock = context.memoryNotes ? context.memoryNotes.trim() : "";
12691265
const notifHint = (context.unreadNotificationCount ?? 0) > 0
@@ -1290,16 +1286,18 @@ export async function streamShipBrainQuestion(input: {
12901286
)
12911287
];
12921288

1293-
// Invoke non-streaming first to check for tool calls
1289+
// Single non-streaming call to detect tool calls, reusing response text to avoid a second LLM round-trip
12941290
const probeModel = getModel({ temperature: 0.2, streaming: false });
12951291
const probeWithTools = probeModel.bind({ tools: getLangChainToolSpecs() } as any);
12961292
const probeResponse = await probeWithTools.invoke(messages) as AIMessage;
12971293
const toolCalls = (probeResponse as any).tool_calls ?? (probeResponse as any).additional_kwargs?.tool_calls ?? [];
12981294

12991295
if (!toolCalls.length) {
1300-
// No tool call — stream the response
1301-
const stream = await modelWithTools.stream(messages);
1302-
return { ...base, action: null, stream, responseSource: "foundry_iq" };
1296+
// Reuse probe response directly — no second LLM call needed
1297+
const content = typeof probeResponse.content === "string"
1298+
? probeResponse.content
1299+
: JSON.stringify(probeResponse.content);
1300+
return { ...base, action: null, stream: textStream(content), responseSource: "foundry_iq" };
13031301
}
13041302

13051303
// Parse tool call

0 commit comments

Comments
 (0)