From 91460f69051b435eaf32d85c2c66cb85fbcdd98d Mon Sep 17 00:00:00 2001 From: cpf Date: Sun, 21 Sep 2025 23:35:09 +0800 Subject: [PATCH 01/16] feat: Gemini API translation and streaming integration (squashed) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scope: /src/routes/generate-content/* only 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- src/routes/generate-content/handler.ts | 335 +++++++++ src/routes/generate-content/route.ts | 58 ++ src/routes/generate-content/translation.ts | 756 +++++++++++++++++++++ src/routes/generate-content/types.ts | 145 ++++ src/routes/generate-content/utils.ts | 50 ++ 5 files changed, 1344 insertions(+) create mode 100644 src/routes/generate-content/handler.ts create mode 100644 src/routes/generate-content/route.ts create mode 100644 src/routes/generate-content/translation.ts create mode 100644 src/routes/generate-content/types.ts create mode 100644 src/routes/generate-content/utils.ts diff --git a/src/routes/generate-content/handler.ts b/src/routes/generate-content/handler.ts new file mode 100644 index 000000000..0f5d93dc8 --- /dev/null +++ b/src/routes/generate-content/handler.ts @@ -0,0 +1,335 @@ +import type { Context } from "hono" +import type { SSEStreamingApi } from "hono/streaming" + +import { streamSSE } from "hono/streaming" + +import { awaitApproval } from "~/lib/approval" +import { checkRateLimit } from "~/lib/rate-limit" +import { state } from "~/lib/state" +import { getTokenCount } from "~/lib/tokenizer" +import { + createChatCompletions, + type ChatCompletionResponse, + type ChatCompletionChunk, +} from "~/services/copilot/create-chat-completions" + +// Helper function to extract model from URL path +function extractModelFromUrl(url: string): string { + const match = url.match(/\/v1beta\/models\/([^:]+):/) + if (!match) { + throw new Error("Model name is required in URL path") + } + return match[1] +} + +import { + translateGeminiToOpenAINonStream, + translateGeminiToOpenAIStream, + translateOpenAIToGemini, + translateGeminiCountTokensToOpenAI, + translateTokenCountToGemini, + translateOpenAIChunkToGemini, +} from "./translation" +import { + type GeminiRequest, + type GeminiCountTokensRequest, + type GeminiStreamResponse, + type GeminiResponse, +} from "./types" + +// Standard generation endpoint +export async function handleGeminiGeneration(c: Context) { + const model = extractModelFromUrl(c.req.url) + + if (!model) { + throw new Error("Model name is required in URL path") + } + + await checkRateLimit(state) + + const geminiPayload = await c.req.json() + + const openAIPayload = translateGeminiToOpenAINonStream(geminiPayload, model) + + if (state.manualApprove) { + await awaitApproval() + } + + const response = await createChatCompletions(openAIPayload) + + if (isNonStreaming(response)) { + const geminiResponse = translateOpenAIToGemini(response) + + return c.json(geminiResponse) + } + + // This shouldn't happen for non-streaming endpoint + throw new Error("Unexpected streaming response for non-streaming endpoint") +} + +// Helper function to handle non-streaming response conversion +function handleNonStreamingToStreaming( + c: Context, + geminiResponse: GeminiResponse, +) { + return streamSSE(c, async (stream) => { + try { + const firstPart = geminiResponse.candidates[0]?.content?.parts?.[0] + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + const hasTextContent = firstPart && "text" in firstPart + + // eslint-disable-next-line unicorn/prefer-ternary + if (hasTextContent) { + await sendTextInChunks(stream, firstPart.text, geminiResponse) + } else { + await sendFallbackResponse(stream, geminiResponse) + } + + // Add a small delay to ensure all data is flushed + await new Promise((resolve) => setTimeout(resolve, 50)) + } catch (error) { + console.error("[GEMINI_STREAM] Error in non-streaming conversion", error) + } finally { + try { + await stream.close() + } catch (closeError) { + console.error( + "[GEMINI_STREAM] Error closing non-streaming conversion stream", + closeError, + ) + } + } + }) +} + +// Helper function to send text in chunks with configuration object +async function sendTextInChunks( + stream: SSEStreamingApi, + text: string, + geminiResponse: GeminiResponse, +) { + const chunkSize = Math.max(1, Math.min(50, text.length)) + let lastWritePromise: Promise = Promise.resolve() + + for (let i = 0; i < text.length; i += chunkSize) { + const chunk = text.slice(i, i + chunkSize) + const isLast = i + chunkSize >= text.length + const streamResponse: GeminiStreamResponse = { + candidates: [ + { + content: { + parts: [{ text: chunk }], + role: "model", + }, + finishReason: + isLast ? geminiResponse.candidates[0]?.finishReason : undefined, + index: 0, + }, + ], + ...(isLast && geminiResponse.usageMetadata ? + { usageMetadata: geminiResponse.usageMetadata } + : {}), + } + + // Wait for previous write to complete before writing new chunk + await lastWritePromise + lastWritePromise = stream.writeSSE({ + data: JSON.stringify(streamResponse), + }) + } + + // Wait for final write to complete + await lastWritePromise +} + +// Helper function to send fallback response +async function sendFallbackResponse( + stream: SSEStreamingApi, + geminiResponse: GeminiResponse, +) { + const streamResponse: GeminiStreamResponse = { + candidates: geminiResponse.candidates, + usageMetadata: geminiResponse.usageMetadata, + } + + await stream.writeSSE({ data: JSON.stringify(streamResponse) }) +} + +// Accumulative JSON parser for handling incomplete chunks (based on LiteLLM research) +class StreamingJSONParser { + private accumulatedData = "" + private parseMode: "direct" | "accumulated" = "direct" + + parseChunk(rawData: string): unknown { + if (this.parseMode === "direct") { + try { + return JSON.parse(rawData) + } catch { + // Switch to accumulated mode on first failure (LiteLLM pattern) + this.parseMode = "accumulated" + this.accumulatedData = rawData + return null + } + } else { + // Accumulated mode - keep building until valid JSON + this.accumulatedData += rawData + try { + const result = JSON.parse(this.accumulatedData) as unknown + // Success - reset for next chunk + this.accumulatedData = "" + this.parseMode = "direct" // Can switch back to direct mode + return result + } catch { + // Continue accumulating + return null + } + } + } +} + +// Global parser instance for the stream +// let streamParser = new StreamingJSONParser() + +// Helper function to process chunk and write to stream +async function processAndWriteChunk(params: { + rawEvent: { data?: string } + stream: SSEStreamingApi + lastWritePromise: Promise + streamParser: StreamingJSONParser +}): Promise<{ newWritePromise: Promise; hasFinishReason: boolean }> { + const { rawEvent, stream, lastWritePromise, streamParser } = params + + if (!rawEvent.data) { + return { newWritePromise: lastWritePromise, hasFinishReason: false } + } + + try { + const chunk = streamParser.parseChunk(rawEvent.data) + + // If parser returns null, we're still accumulating + if (!chunk) { + return { newWritePromise: lastWritePromise, hasFinishReason: false } + } + + const geminiChunk = translateOpenAIChunkToGemini( + chunk as ChatCompletionChunk, + ) + + if (geminiChunk) { + // Check if this chunk contains a finish reason + const chunkHasFinishReason = geminiChunk.candidates.some( + (c) => c.finishReason && c.finishReason !== "FINISH_REASON_UNSPECIFIED", + ) + + // Wait for previous write to complete before writing new chunk + await lastWritePromise + const newWritePromise = stream.writeSSE({ + data: JSON.stringify(geminiChunk), + }) + + return { newWritePromise, hasFinishReason: chunkHasFinishReason } + } else { + return { newWritePromise: lastWritePromise, hasFinishReason: false } + } + } catch (parseError) { + console.error("[GEMINI_STREAM] Error parsing chunk", parseError) + return { newWritePromise: lastWritePromise, hasFinishReason: false } + } +} + +// Helper function to handle streaming response processing +function handleStreamingResponse( + c: Context, + response: AsyncIterable<{ data?: string }>, +) { + return streamSSE(c, async (stream) => { + // Create a parser instance for this stream (each request gets its own parser) + const streamParser = new StreamingJSONParser() + let lastWritePromise: Promise = Promise.resolve() + + try { + for await (const rawEvent of response) { + if (rawEvent.data === "[DONE]") { + break + } + + const result = await processAndWriteChunk({ + rawEvent, + stream, + lastWritePromise, + streamParser, + }) + lastWritePromise = result.newWritePromise + } + + // Wait for all writes to complete before closing + await lastWritePromise + + // Add a small delay to ensure all data is flushed + await new Promise((resolve) => setTimeout(resolve, 50)) + } catch (error) { + console.error("[GEMINI_STREAM] Error in streaming processing", error) + // Ensure we don't leave the stream hanging + } finally { + // Always close the stream, but with proper cleanup + try { + await stream.close() + } catch (closeError) { + console.error("[GEMINI_STREAM] Error closing stream", closeError) + } + } + }) +} + +// Streaming generation endpoint +export async function handleGeminiStreamGeneration(c: Context) { + const model = extractModelFromUrl(c.req.url) + + if (!model) { + throw new Error("Model name is required in URL path") + } + + await checkRateLimit(state) + + const geminiPayload = await c.req.json() + + const openAIPayload = translateGeminiToOpenAIStream(geminiPayload, model) + + if (state.manualApprove) { + await awaitApproval() + } + + const response = await createChatCompletions(openAIPayload) + + if (isNonStreaming(response)) { + const geminiResponse = translateOpenAIToGemini(response) + + return handleNonStreamingToStreaming(c, geminiResponse) + } + + return handleStreamingResponse(c, response) +} + +// Token counting endpoint +export async function handleGeminiCountTokens(c: Context) { + const model = extractModelFromUrl(c.req.url) + + if (!model) { + throw new Error("Model name is required in URL path") + } + + const geminiPayload = await c.req.json() + + const openAIPayload = translateGeminiCountTokensToOpenAI(geminiPayload, model) + + const tokenCounts = getTokenCount(openAIPayload.messages) + + const totalTokens = tokenCounts.input + tokenCounts.output + const geminiResponse = translateTokenCountToGemini(totalTokens) + + return c.json(geminiResponse) +} + +const isNonStreaming = ( + response: Awaited>, +): response is ChatCompletionResponse => "choices" in response diff --git a/src/routes/generate-content/route.ts b/src/routes/generate-content/route.ts new file mode 100644 index 000000000..ca57bda56 --- /dev/null +++ b/src/routes/generate-content/route.ts @@ -0,0 +1,58 @@ +import { Hono } from "hono" + +import { forwardError } from "~/lib/error" + +import { + handleGeminiGeneration, + handleGeminiStreamGeneration, + handleGeminiCountTokens, +} from "./handler" + +const router = new Hono() + +// Streaming generation endpoint +// POST /v1beta/models/{model}:streamGenerateContent +router.post("/v1beta/models/*", async (c, next) => { + const url = c.req.url + if (url.includes(":streamGenerateContent")) { + try { + return await handleGeminiStreamGeneration(c) + } catch (error) { + return await forwardError(c, error) + } + } + await next() +}) + +// Token counting endpoint +// POST /v1beta/models/{model}:countTokens +router.post("/v1beta/models/*", async (c, next) => { + const url = c.req.url + if (url.includes(":countTokens")) { + try { + return await handleGeminiCountTokens(c) + } catch (error) { + return await forwardError(c, error) + } + } + await next() +}) + +// Standard generation endpoint +// POST /v1beta/models/{model}:generateContent +router.post("/v1beta/models/*", async (c, next) => { + const url = c.req.url + if ( + url.includes(":generateContent") + && !url.includes(":streamGenerateContent") + ) { + try { + return await handleGeminiGeneration(c) + } catch (error) { + return await forwardError(c, error) + } + } + await next() +}) + +export { router as geminiRouter } diff --git a/src/routes/generate-content/translation.ts b/src/routes/generate-content/translation.ts new file mode 100644 index 000000000..7adc9a3cb --- /dev/null +++ b/src/routes/generate-content/translation.ts @@ -0,0 +1,756 @@ +import { + type ChatCompletionResponse, + type ChatCompletionChunk, + type ChatCompletionsPayload, + type ContentPart, + type Message, + type Tool, + type ToolCall, +} from "~/services/copilot/create-chat-completions" + +import { + type GeminiRequest, + type GeminiResponse, + type GeminiContent, + type GeminiPart, + type GeminiTextPart, + type GeminiFunctionCallPart, + type GeminiFunctionResponsePart, + type GeminiTool, + type GeminiCandidate, + type GeminiCountTokensRequest, + type GeminiCountTokensResponse, + type GeminiUsageMetadata, +} from "./types" +import { mapOpenAIFinishReasonToGemini } from "./utils" + +// Model mapping for Gemini models - only map unsupported variants to supported ones +function mapGeminiModelToCopilot(geminiModel: string): string { + const modelMap: Record = { + "gemini-2.5-flash": "gemini-2.0-flash-001", // Map to supported Gemini model + "gemini-2.0-flash": "gemini-2.0-flash-001", // Map to full model name + } + + return modelMap[geminiModel] || geminiModel // Return original if supported +} + +// Request translation: Gemini -> OpenAI + +export function translateGeminiToOpenAINonStream( + payload: GeminiRequest, + model: string, +): ChatCompletionsPayload { + const tools = + translateGeminiToolsToOpenAI(payload.tools) + || synthesizeToolsFromContents(payload.contents) + const result = { + model: mapGeminiModelToCopilot(model), + messages: translateGeminiContentsToOpenAI( + payload.contents, + payload.systemInstruction, + ), + max_tokens: payload.generationConfig?.maxOutputTokens || 4096, + stop: payload.generationConfig?.stopSequences, + stream: false, + temperature: payload.generationConfig?.temperature, + top_p: payload.generationConfig?.topP, + tools, + tool_choice: + tools ? translateGeminiToolConfigToOpenAI(payload.toolConfig) : undefined, + } + + return result +} + +export function translateGeminiToOpenAIStream( + payload: GeminiRequest, + model: string, +): ChatCompletionsPayload { + const tools = + translateGeminiToolsToOpenAI(payload.tools) + || synthesizeToolsFromContents(payload.contents) + const result = { + model: mapGeminiModelToCopilot(model), + messages: translateGeminiContentsToOpenAI( + payload.contents, + payload.systemInstruction, + ), + max_tokens: payload.generationConfig?.maxOutputTokens || 4096, + stop: payload.generationConfig?.stopSequences, + stream: true, + temperature: payload.generationConfig?.temperature, + top_p: payload.generationConfig?.topP, + tools, + tool_choice: + tools ? translateGeminiToolConfigToOpenAI(payload.toolConfig) : undefined, + } + + return result +} + +// Helper function to process function response arrays +function processFunctionResponseArray( + responseArray: Array<{ + functionResponse: { name: string; response: unknown } + }>, + pendingToolCalls: Map, + messages: Array, +): void { + for (const responseItem of responseArray) { + if ("functionResponse" in responseItem) { + const functionName = responseItem.functionResponse.name + const toolCallId = pendingToolCalls.get(functionName) + if (toolCallId) { + messages.push({ + role: "tool", + tool_call_id: toolCallId, + content: JSON.stringify(responseItem.functionResponse.response), + }) + pendingToolCalls.delete(functionName) + } + } + } +} + +// Helper function to check if tool calls have corresponding tool responses +function hasCorrespondingToolResponses( + messages: Array, + toolCalls: Array, +): boolean { + const toolCallIds = new Set(toolCalls.map((call) => call.id)) + + // Look for tool messages that respond to these tool calls + for (const message of messages) { + if (message.role === "tool" && message.tool_call_id) { + toolCallIds.delete(message.tool_call_id) + } + } + + // If any tool call ID remains, it means there's no corresponding response + return toolCallIds.size === 0 +} + +// Helper function to process function responses in content +function processFunctionResponses( + functionResponses: Array, + pendingToolCalls: Map, + messages: Array, +): void { + for (const funcResponse of functionResponses) { + const functionName = funcResponse.functionResponse.name + const toolCallId = pendingToolCalls.get(functionName) + if (toolCallId) { + messages.push({ + role: "tool", + tool_call_id: toolCallId, + content: JSON.stringify(funcResponse.functionResponse.response), + }) + pendingToolCalls.delete(functionName) + } + } +} + +// Helper function to process function calls and create assistant message +function processFunctionCalls(options: { + functionCalls: Array + content: GeminiContent + pendingToolCalls: Map + messages: Array +}): void { + const { functionCalls, content, pendingToolCalls, messages } = options + + const textContent = extractTextFromGeminiContent(content) + const toolCalls = functionCalls.map((call) => { + const toolCallId = generateToolCallId(call.functionCall.name) + // Remember this tool call for later matching with responses + pendingToolCalls.set(call.functionCall.name, toolCallId) + + return { + id: toolCallId, + type: "function" as const, + function: { + name: call.functionCall.name, + arguments: JSON.stringify(call.functionCall.args), + }, + } + }) + + messages.push({ + role: "assistant", + content: textContent || null, + tool_calls: toolCalls, + }) +} + +// Helper function to merge consecutive messages with same role +function mergeConsecutiveSameRoleMessages( + messages: Array, +): Array { + const mergedMessages: Array = [] + for (const message of messages) { + const lastMessage = mergedMessages.at(-1) + + if ( + lastMessage + && lastMessage.role === message.role + && !lastMessage.tool_calls + && !message.tool_calls + ) { + // Merge with previous message of same role + if ( + typeof lastMessage.content === "string" + && typeof message.content === "string" + ) { + lastMessage.content = lastMessage.content + "\n\n" + message.content + } else { + // Can't merge complex content, keep separate + mergedMessages.push(message) + } + } else { + // Add content validation for user messages (based on LiteLLM research) + if ( + message.role === "user" + && typeof message.content === "string" + && !message.content.trim() + ) { + message.content = " " // Add minimal text content as fallback + } + mergedMessages.push(message) + } + } + return mergedMessages +} + +// Helper function to remove incomplete assistant messages +function removeIncompleteAssistantMessages(messages: Array): void { + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i] + if ( + message.role === "assistant" + && message.tool_calls + && !hasCorrespondingToolResponses(messages, message.tool_calls) + ) { + messages.splice(i, 1) + } + } +} + +function translateGeminiContentsToOpenAI( + contents: Array< + | GeminiContent + | Array<{ + functionResponse: { id?: string; name: string; response: unknown } + }> + >, + systemInstruction?: GeminiContent, +): Array { + const messages: Array = [] + const pendingToolCalls = new Map() // function name -> tool_call_id + + // Add system instruction first if present + if (systemInstruction) { + const systemText = extractTextFromGeminiContent(systemInstruction) + if (systemText) { + messages.push({ role: "system", content: systemText }) + } + } + + // Process conversation contents + for (const item of contents) { + // Handle special case where Gemini CLI sends function responses as nested arrays + if (Array.isArray(item)) { + processFunctionResponseArray(item, pendingToolCalls, messages) + continue + } + + const content = item + const role = content.role === "model" ? "assistant" : "user" + + // Check for function calls/responses + const functionCalls = content.parts.filter( + (part): part is GeminiFunctionCallPart => "functionCall" in part, + ) + const functionResponses = content.parts.filter( + (part): part is GeminiFunctionResponsePart => "functionResponse" in part, + ) + + if (functionResponses.length > 0) { + processFunctionResponses(functionResponses, pendingToolCalls, messages) + } + + if (functionCalls.length > 0 && role === "assistant") { + processFunctionCalls({ + functionCalls, + content, + pendingToolCalls, + messages, + }) + } else { + // Regular message + const messageContent = translateGeminiContentToOpenAI(content) + if (messageContent) { + messages.push({ role, content: messageContent }) + } + } + } + + // Post-process: Remove incomplete assistant messages from cancelled tool calls + removeIncompleteAssistantMessages(messages) + + // Post-process: Merge consecutive messages with same role (based on LiteLLM research) + return mergeConsecutiveSameRoleMessages(messages) +} + +function synthesizeToolsFromContents( + contents: Array< + | GeminiContent + | Array<{ + functionResponse: { id?: string; name: string; response: unknown } + }> + >, +): Array | undefined { + const names = new Set() + for (const item of contents) { + if (Array.isArray(item)) continue + for (const part of item.parts) { + if ("functionCall" in part && part.functionCall.name) { + names.add(part.functionCall.name) + } + } + } + if (names.size === 0) return undefined + return Array.from(names).map((name) => ({ + type: "function", + function: { name, parameters: { type: "object", properties: {} } }, + })) +} + +function translateGeminiContentToOpenAI( + content: GeminiContent, +): string | Array | null { + if (content.parts.length === 0) return null + + const hasMedia = content.parts.some((part) => "inlineData" in part) + + if (!hasMedia) { + // Text-only content + return extractTextFromGeminiContent(content) + } + + // Mixed content with media + const contentParts: Array = [] + for (const part of content.parts) { + if ("text" in part) { + contentParts.push({ type: "text", text: part.text }) + } else if ("inlineData" in part) { + contentParts.push({ + type: "image_url", + image_url: { + url: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`, + }, + }) + } + } + + return contentParts +} + +function extractTextFromGeminiContent(content: GeminiContent): string { + return content.parts + .filter((part): part is GeminiTextPart => "text" in part) + .map((part) => part.text) + .join("\n\n") +} + +function translateGeminiToolsToOpenAI( + geminiTools?: Array, +): Array | undefined { + if (!geminiTools || geminiTools.length === 0) return undefined + + const tools: Array = [] + for (const tool of geminiTools) { + // Handle standard function declarations + if (tool.functionDeclarations) { + for (const func of tool.functionDeclarations) { + // Validate that function name exists and is not empty + if ( + !func.name + || typeof func.name !== "string" + || func.name.trim() === "" + ) { + continue + } + + // Ensure parameters is always a valid object + + const validParameters = func.parametersJsonSchema + || func.parameters || { type: "object", properties: {} } + + tools.push({ + type: "function", + function: { + name: func.name, + description: func.description, + parameters: validParameters, + }, + }) + } + } + + // Handle googleSearch tool (special case) + if (tool.googleSearch !== undefined) { + tools.push({ + type: "function", + function: { + name: "google_web_search", + description: + "Performs a web search using Google Search (via the Gemini API) and returns the results. This tool is useful for finding information on the internet based on a query.", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "The search query to find information on the web.", + }, + }, + required: ["query"], + }, + }, + }) + } + + // Handle urlContext tool (special case for web_fetch) + // Note: GitHub Copilot API doesn't support web_fetch functionality + // Skip this tool to avoid "Failed to create chat completions" errors + if (tool.urlContext !== undefined) { + continue + } + } + + return tools.length > 0 ? tools : undefined +} + +function translateGeminiToolConfigToOpenAI( + toolConfig?: GeminiRequest["toolConfig"], +): ChatCompletionsPayload["tool_choice"] { + if (!toolConfig) return undefined + + const mode = toolConfig.functionCallingConfig.mode + switch (mode) { + case "AUTO": { + return "auto" + } + case "ANY": { + return "required" + } + case "NONE": { + return "none" + } + default: { + return undefined + } + } +} + +// Response translation: OpenAI -> Gemini + +export function translateOpenAIToGemini( + response: ChatCompletionResponse, +): GeminiResponse { + const candidates: Array = response.choices.map( + (choice, index) => ({ + content: translateOpenAIMessageToGeminiContent(choice.message), + finishReason: mapOpenAIFinishReasonToGemini(choice.finish_reason), + index, + }), + ) + + return { + candidates, + usageMetadata: { + promptTokenCount: response.usage?.prompt_tokens || 0, + candidatesTokenCount: response.usage?.completion_tokens || 0, + totalTokenCount: response.usage?.total_tokens || 0, + }, + } +} + +function translateOpenAIMessageToGeminiContent( + message: Message, +): GeminiContent { + const parts: Array = [] + + // Handle text content + if (typeof message.content === "string") { + if (message.content) { + parts.push({ text: message.content }) + } + } else if (Array.isArray(message.content)) { + for (const part of message.content) { + if (part.type === "text") { + parts.push({ text: part.text }) + } else { + // Convert data URL back to inline data + const match = part.image_url.url.match(/^data:([^;]+);base64,(.+)$/) + if (match) { + parts.push({ + inlineData: { + mimeType: match[1], + data: match[2], + }, + }) + } + } + } + } + + // Handle tool calls + if (message.tool_calls) { + for (const toolCall of message.tool_calls) { + parts.push({ + functionCall: { + name: toolCall.function.name, + args: + toolCall.function.arguments ? + (JSON.parse(toolCall.function.arguments) as Record< + string, + unknown + >) + : {}, + }, + }) + } + } + + return { + parts, + role: "model", + } +} + +// Utility functions + +function generateToolCallId(functionName: string): string { + return `call_${functionName}_${Date.now()}_${Math.random().toString(36).slice(2, 11)}` +} + +// Helper function to process tool calls in streaming chunks +function processToolCalls( + toolCalls: Array<{ + index: number + id?: string + type?: "function" + function?: { + name?: string + arguments?: string + } + }>, +): Array { + const parts: Array = [] + + for (const toolCall of toolCalls) { + if (!toolCall.function?.name) { + continue + } + + let args: Record + try { + args = JSON.parse(toolCall.function.arguments || "{}") as Record< + string, + unknown + > + } catch { + // In streaming, arguments might be incomplete JSON + // Skip this chunk and wait for complete arguments + continue + } + + parts.push({ + functionCall: { + name: toolCall.function.name, + args, + }, + }) + } + + return parts +} + +// Helper function to create usage metadata +function createUsageMetadata(chunk: ChatCompletionChunk): GeminiUsageMetadata { + return { + promptTokenCount: chunk.usage?.prompt_tokens || 0, + candidatesTokenCount: chunk.usage?.completion_tokens || 0, + totalTokenCount: chunk.usage?.total_tokens || 0, + } +} + +// Helper function to process chunk parts +function processChunkParts(choice: { + delta: { + content?: string | null + tool_calls?: Array<{ + index: number + id?: string + type?: "function" + function?: { + name?: string + arguments?: string + } + }> + } +}): Array { + const parts: Array = [] + + if (choice.delta.content) { + parts.push({ text: choice.delta.content }) + } + + if (choice.delta.tool_calls) { + parts.push(...processToolCalls(choice.delta.tool_calls)) + } + + return parts +} + +// Helper function to determine finish reason inclusion +function shouldIncludeFinishReason(choice: { + finish_reason: "stop" | "length" | "tool_calls" | "content_filter" | null + delta: { + tool_calls?: Array + } +}): boolean { + // Always include finish_reason when present, regardless of tool calls + // This ensures proper stream termination for both text and tool call completions + return Boolean(choice.finish_reason) +} + +// Helper function to create candidate object +function createGeminiCandidate( + parts: Array, + mappedFinishReason: string | undefined, + index: number, +): GeminiCandidate { + return { + content: { + parts, + role: "model", + }, + finishReason: mappedFinishReason as GeminiCandidate["finishReason"], + index, + } +} + +// Helper function to handle parts processing and validation +function processParts(choice: { + finish_reason: "stop" | "length" | "tool_calls" | "content_filter" | null + delta: { + content?: string | null + tool_calls?: Array<{ + index: number + id?: string + type?: "function" + function?: { + name?: string + arguments?: string + } + }> + } +}): Array | null { + const parts = processChunkParts(choice) + + if (parts.length === 0 && !choice.finish_reason) { + return null + } + + // If we have a finish reason but no parts, add an empty text part + // This ensures Gemini CLI receives a properly formatted completion chunk + if (parts.length === 0 && choice.finish_reason) { + parts.push({ text: "" }) + } + + return parts +} + +// Helper function to build complete response +function buildGeminiResponse( + candidate: GeminiCandidate, + shouldInclude: boolean, + chunk: ChatCompletionChunk, +): { + candidates: Array + usageMetadata?: GeminiUsageMetadata +} { + const response: { + candidates: Array + usageMetadata?: GeminiUsageMetadata + } = { + candidates: [candidate], + } + + if (shouldInclude) { + response.usageMetadata = createUsageMetadata(chunk) + } + + return response +} + +// Stream translation: OpenAI Chunk -> Gemini Stream Response +export function translateOpenAIChunkToGemini(chunk: ChatCompletionChunk): { + candidates: Array + usageMetadata?: GeminiUsageMetadata +} | null { + if (chunk.choices.length === 0) { + return null + } + + const choice = chunk.choices[0] + + const parts = processParts(choice) + if (!parts) { + return null + } + + const shouldInclude = shouldIncludeFinishReason(choice) + const mappedFinishReason = + shouldInclude ? + mapOpenAIFinishReasonToGemini(choice.finish_reason) + : undefined + + const candidate = createGeminiCandidate( + parts, + mappedFinishReason, + choice.index, + ) + const response = buildGeminiResponse(candidate, shouldInclude, chunk) + + return response +} + +// Token counting translation + +export function translateGeminiCountTokensToOpenAI( + request: GeminiCountTokensRequest, + model: string, +): ChatCompletionsPayload { + const tools = + translateGeminiToolsToOpenAI(request.tools) + || synthesizeToolsFromContents(request.contents) + return { + model: mapGeminiModelToCopilot(model), + messages: translateGeminiContentsToOpenAI( + request.contents, + request.systemInstruction, + ), + max_tokens: 1, + tools, + } +} + +export function translateTokenCountToGemini( + totalTokens: number, +): GeminiCountTokensResponse { + return { + totalTokens, + } +} diff --git a/src/routes/generate-content/types.ts b/src/routes/generate-content/types.ts new file mode 100644 index 000000000..8de67138d --- /dev/null +++ b/src/routes/generate-content/types.ts @@ -0,0 +1,145 @@ +// Gemini API Types + +export interface GeminiRequest { + contents: Array + tools?: Array + toolConfig?: GeminiToolConfig + safetySettings?: Array + systemInstruction?: GeminiContent + generationConfig?: GeminiGenerationConfig +} + +export interface GeminiContent { + parts: Array + role?: "user" | "model" +} + +export type GeminiPart = + | GeminiTextPart + | GeminiInlineDataPart + | GeminiFunctionCallPart + | GeminiFunctionResponsePart + +export interface GeminiTextPart { + text: string +} + +export interface GeminiInlineDataPart { + inlineData: { + mimeType: string + data: string + } +} + +export interface GeminiFunctionCallPart { + functionCall: { + name: string + args: Record + } +} + +export interface GeminiFunctionResponsePart { + functionResponse: { + name: string + response: Record + } +} + +export interface GeminiTool { + functionDeclarations?: Array + googleSearch?: Record + urlContext?: Record +} + +export interface GeminiFunctionDeclaration { + name: string + description?: string + parameters?: Record + parametersJsonSchema?: Record +} + +export interface GeminiToolConfig { + functionCallingConfig: { + mode: "AUTO" | "ANY" | "NONE" + allowedFunctionNames?: Array + } +} + +export interface GeminiSafetySetting { + category: string + threshold: string +} + +export interface GeminiGenerationConfig { + stopSequences?: Array + temperature?: number + maxOutputTokens?: number + topP?: number + topK?: number +} + +// Response types +export interface GeminiResponse { + candidates: Array + usageMetadata?: GeminiUsageMetadata + promptFeedback?: GeminiPromptFeedback +} + +export interface GeminiPromptFeedback { + blockReason?: + | "BLOCK_REASON_UNSPECIFIED" + | "SAFETY" + | "OTHER" + | "BLOCKLIST" + | "PROHIBITED_CONTENT" + safetyRatings?: Array +} + +export interface GeminiCandidate { + content: GeminiContent + finishReason?: + | "FINISH_REASON_UNSPECIFIED" + | "STOP" + | "MAX_TOKENS" + | "SAFETY" + | "RECITATION" + | "LANGUAGE" + | "OTHER" + | "BLOCKLIST" + | "PROHIBITED_CONTENT" + | "SPII" + | "MALFORMED_FUNCTION_CALL" + | "IMAGE_SAFETY" + | "UNEXPECTED_TOOL_CALL" + | "TOO_MANY_TOOL_CALLS" + index: number + safetyRatings?: Array +} + +export interface GeminiSafetyRating { + category: string + probability: string +} + +export interface GeminiUsageMetadata { + promptTokenCount: number + candidatesTokenCount: number + totalTokenCount: number +} + +// Token counting types +export interface GeminiCountTokensRequest { + contents: Array + tools?: Array + systemInstruction?: GeminiContent +} + +export interface GeminiCountTokensResponse { + totalTokens: number +} + +// Streaming types +export interface GeminiStreamResponse { + candidates?: Array + usageMetadata?: GeminiUsageMetadata +} diff --git a/src/routes/generate-content/utils.ts b/src/routes/generate-content/utils.ts new file mode 100644 index 000000000..88f1c89b5 --- /dev/null +++ b/src/routes/generate-content/utils.ts @@ -0,0 +1,50 @@ +import { type GeminiCandidate } from "./types" + +export function mapOpenAIFinishReasonToGemini( + finishReason: string | null, +): GeminiCandidate["finishReason"] { + switch (finishReason) { + case "stop": { + return "STOP" + } + case "length": { + return "MAX_TOKENS" + } + case "content_filter": { + return "SAFETY" + } + case "tool_calls": { + return "STOP" // Gemini doesn't have a specific tool_calls finish reason, map to STOP + } + default: { + return "FINISH_REASON_UNSPECIFIED" + } + } +} + +// Add the reverse mapping - Gemini → OpenAI (based on LiteLLM research) +export function mapGeminiFinishReasonToOpenAI( + finishReason: string | undefined, +): "stop" | "length" | "content_filter" | "tool_calls" { + switch (finishReason) { + case "STOP": + case "FINISH_REASON_UNSPECIFIED": + case "MALFORMED_FUNCTION_CALL": { + return "stop" + } + case "MAX_TOKENS": { + return "length" + } + case "SAFETY": + case "RECITATION": + case "BLOCKLIST": + case "PROHIBITED_CONTENT": + case "SPII": + case "IMAGE_SAFETY": { + return "content_filter" + } + default: { + return "stop" + } + } +} From fa6b6124648908e91741e7694a3e2420ebf140b4 Mon Sep 17 00:00:00 2001 From: cpf Date: Sun, 21 Sep 2025 23:38:40 +0800 Subject: [PATCH 02/16] lint --- src/routes/messages/gemini-handler.ts | 627 ++++++++++++++++++++++ src/routes/messages/gemini-route.ts | 47 ++ src/routes/messages/gemini-translation.ts | 439 +++++++++++++++ src/routes/messages/gemini-types.ts | 123 +++++ 4 files changed, 1236 insertions(+) create mode 100644 src/routes/messages/gemini-handler.ts create mode 100644 src/routes/messages/gemini-route.ts create mode 100644 src/routes/messages/gemini-translation.ts create mode 100644 src/routes/messages/gemini-types.ts diff --git a/src/routes/messages/gemini-handler.ts b/src/routes/messages/gemini-handler.ts new file mode 100644 index 000000000..b3db02409 --- /dev/null +++ b/src/routes/messages/gemini-handler.ts @@ -0,0 +1,627 @@ +import type { Context } from "hono" +import type { SSEStreamingApi } from "hono/streaming" + +import consola from "consola" +import { streamSSE } from "hono/streaming" +import { promises as fs } from "node:fs" +import path from "node:path" + +import { awaitApproval } from "~/lib/approval" +import { checkRateLimit } from "~/lib/rate-limit" +import { state } from "~/lib/state" +import { getTokenCount } from "~/lib/tokenizer" +import { + createChatCompletions, + type ChatCompletionChunk, + type ChatCompletionResponse, +} from "~/services/copilot/create-chat-completions" + +import { + translateGeminiToOpenAINonStream, + translateGeminiToOpenAIStream, + translateOpenAIToGemini, + translateGeminiCountTokensToOpenAI, + translateTokenCountToGemini, + translateOpenAIChunkToGemini, +} from "./gemini-translation" +import { + type GeminiRequest, + type GeminiCountTokensRequest, + type GeminiStreamResponse, + type GeminiResponse, +} from "./gemini-types" + +// Debug logging interface +interface GeminiDebugLog { + timestamp: string + type: + | "request" + | "response" + | "translation" + | "error" + | "stream_chunk" + | "stream_translation" + endpoint: string + data: unknown + copilotRequest?: unknown + copilotResponse?: unknown + finalResponse?: unknown +} + +// File logging functions +async function writeLogToFile(logEntry: GeminiDebugLog) { + const logsDir = path.join(process.cwd(), "logs") + + try { + // Ensure logs directory exists + await fs.mkdir(logsDir, { recursive: true }) + + const logLine = JSON.stringify(logEntry) + "\n" + + // Write to main debug log + await fs.appendFile(path.join(logsDir, "gemini-debug.log"), logLine) + + // Write to specific logs based on type + if (logEntry.type === "error") { + await fs.appendFile(path.join(logsDir, "gemini-errors.log"), logLine) + } else if ( + logEntry.type === "translation" + || logEntry.type === "stream_translation" + ) { + await fs.appendFile(path.join(logsDir, "gemini-translation.log"), logLine) + } + } catch (error) { + consola.error("Failed to write log file:", error) + } +} + +// Helper function to truncate data for logging +function truncateData(data: unknown, maxLength = 200): unknown { + if (typeof data === "string") { + return data.length > maxLength ? `${data.slice(0, maxLength)}...` : data + } + + if (Array.isArray(data)) { + return data.map((item) => truncateData(item, maxLength)) + } + + if (data && typeof data === "object") { + const obj = data as Record + const result: Record = {} + + for (const [key, value] of Object.entries(obj)) { + if (key === "messages" && Array.isArray(value)) { + result[key] = value.map((msg: { role: string; content: unknown }) => ({ + role: msg.role, + content: getContentDisplay(msg.content), + })) + } else if (key === "contents" && Array.isArray(value)) { + result[key] = value.map( + (content: { role: string; parts?: Array }) => ({ + role: content.role, + parts: + Array.isArray(content.parts) && content.parts.length > 0 ? + `[${content.parts.length} parts]` + : content.parts, + }), + ) + } else { + result[key] = truncateData(value, maxLength) + } + } + return result + } + + return data +} + +// Helper function to display content for logging +function getContentDisplay(content: unknown): unknown { + if (typeof content === "string") { + return truncateData(content, 100) + } + if (Array.isArray(content) && content.length > 0) { + return `[content array: ${content.length} items]` + } + return content +} + +// Debug logging functions +function logGeminiDebug( + type: string, + endpoint: string, + options: { data: unknown; extra?: Record }, +) { + const { data, extra } = options + const truncatedData = truncateData(data) + const truncatedExtra = extra ? truncateData(extra) : undefined + + const logEntry: GeminiDebugLog = { + timestamp: new Date().toISOString(), + type: type as GeminiDebugLog["type"], + endpoint, + data: truncatedData, + ...(truncatedExtra as Record), + } + + // Console logging - more concise + const endpointPath = new URL(endpoint).pathname + consola.debug(`[GEMINI-${type.toUpperCase()}] ${endpointPath}`) + + // File logging (async, don't wait) - now always write but with truncated data + writeLogToFile(logEntry).catch((error: unknown) => + consola.error("Log file write error:", error), + ) +} + +function logGeminiError(endpoint: string, error: unknown, data?: unknown) { + const truncatedData = data ? truncateData(data) : undefined + + const logEntry: GeminiDebugLog = { + timestamp: new Date().toISOString(), + type: "error", + endpoint, + data: { + error: error instanceof Error ? error.message : String(error), + stack: error instanceof Error ? error.stack : undefined, + data: truncatedData, + }, + } + + // Console logging - more concise + const endpointPath = new URL(endpoint).pathname + consola.error( + `[GEMINI-ERROR] ${endpointPath}: ${error instanceof Error ? error.message : String(error)}`, + ) + + // File logging (async, don't wait) + writeLogToFile(logEntry).catch((logError: unknown) => + consola.error("Log file write error:", logError), + ) +} + +// Helper function to process stream chunk +async function processStreamChunk( + rawEvent: { data?: string }, + endpoint: string, + stream: SSEStreamingApi, +): Promise { + if (rawEvent.data === "[DONE]") { + return false // Signal to stop processing + } + + if (!rawEvent.data) { + return true // Continue processing + } + + try { + const chunkData = JSON.parse(rawEvent.data) as unknown + const chunk = chunkData as ChatCompletionChunk + const geminiResponse = translateOpenAIChunkToGemini(chunk) + + if (geminiResponse) { + consola.debug("Streaming geminiResponse object:", geminiResponse) + const jsonLine = JSON.stringify(geminiResponse) + consola.debug("Streaming JSON line:", jsonLine) + consola.debug("About to send SSE data:", jsonLine.slice(0, 100)) + + // Validate JSON before sending + try { + JSON.parse(jsonLine) + } catch (validateError) { + logGeminiError(endpoint, validateError, { + rawEvent, + context: "JSON validation failed before sending", + jsonLine: jsonLine.slice(0, 200), + }) + return true // Continue processing + } + + await stream.writeSSE({ + data: jsonLine, + }) + return true // Continue processing + } + return true // Continue processing + } catch (chunkError) { + logGeminiError(endpoint, chunkError, { + rawEvent, + context: "JSON.parse failed in stream", + }) + return true // Continue processing + } +} + +// Error handling helper +function getErrorStatusAndMessage(error: unknown): { + status: number + message: string +} { + if (!(error instanceof Error)) { + return { status: 500, message: "Internal server error" } + } + + const errorMappings = [ + { + condition: (err: Error) => + err.name === "RateLimitError" || err.message.includes("rate limit"), + status: 429, + message: "Rate limit exceeded", + }, + { + condition: (err: Error) => + err.name === "ValidationError" || err.message.includes("validation"), + status: 400, + message: "Invalid request", + }, + { + condition: (err: Error) => + err.name === "AuthenticationError" || err.message.includes("auth"), + status: 401, + message: "Authentication failed", + }, + { + condition: (err: Error) => + err.name === "NotFoundError" || err.message.includes("not found"), + status: 404, + message: "Resource not found", + }, + ] + + for (const mapping of errorMappings) { + if (mapping.condition(error)) { + return { status: mapping.status, message: mapping.message } + } + } + + return { status: 500, message: "Internal server error" } +} + +// Standard generation endpoint +export async function handleGeminiGeneration(c: Context) { + const endpoint = c.req.url + const model = c.req.param("model") + + // IMMEDIATE DEBUG: Log that we entered this handler + logGeminiDebug("handler_entry_GENERATION", endpoint, { + data: { + endpoint: endpoint, + model: model, + context: "Entered handleGeminiGeneration handler (NON-STREAMING)", + }, + }) + + try { + await checkRateLimit(state) + + const geminiPayload = await c.req.json() + logGeminiDebug("request", endpoint, { data: geminiPayload }) + + const openAIPayload = translateGeminiToOpenAINonStream(geminiPayload, model) + logGeminiDebug("translation", endpoint, { + data: openAIPayload, + extra: { copilotRequest: openAIPayload }, + }) + + if (state.manualApprove) { + await awaitApproval() + } + + const response = await createChatCompletions(openAIPayload) + + if (isNonStreaming(response)) { + logGeminiDebug("response", endpoint, { + data: response, + extra: { copilotResponse: response }, + }) + + const geminiResponse = translateOpenAIToGemini(response) + logGeminiDebug("translation", endpoint, { + data: geminiResponse, + extra: { finalResponse: geminiResponse }, + }) + + return c.json(geminiResponse) + } + + // This shouldn't happen for non-streaming endpoint + logGeminiError( + endpoint, + new Error("Unexpected streaming response for non-streaming endpoint"), + ) + return c.json({ error: "Internal error" }, 500) + } catch (error) { + logGeminiError(endpoint, error) + const { status, message } = getErrorStatusAndMessage(error) + return c.json({ error: message }, status as 400 | 401 | 404 | 429 | 500) + } +} + +// Helper function to handle non-streaming response conversion +function handleNonStreamingToStreaming( + c: Context, + geminiResponse: GeminiResponse, + endpoint: string, +) { + return streamSSE(c, async (stream) => { + logGeminiDebug("non_streaming_conversion", endpoint, { + data: { + geminiResponse: truncateData(geminiResponse), + context: "Converting non-streaming response to streaming", + }, + }) + + const textContent = geminiResponse.candidates[0]?.content?.parts?.[0] + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + await (textContent && "text" in textContent ? + sendTextInChunks(stream, textContent.text, { + geminiResponse, + endpoint, + }) + : sendFallbackResponse(stream, geminiResponse, endpoint)) + + logGeminiDebug("stream_closing", endpoint, { + data: { context: "Closing non-streaming to streaming conversion" }, + }) + await stream.close() + }) +} + +// Helper function to send text in chunks with configuration object +async function sendTextInChunks( + stream: SSEStreamingApi, + text: string, + options: { geminiResponse: GeminiResponse; endpoint: string }, +) { + const { geminiResponse, endpoint } = options + logGeminiDebug("text_chunking", endpoint, { + data: { + text: text, + textLength: text.length, + context: "Processing text for chunking", + }, + }) + const chunkSize = Math.max(1, Math.min(50, text.length)) + + for (let i = 0; i < text.length; i += chunkSize) { + const chunk = text.slice(i, i + chunkSize) + const isLast = i + chunkSize >= text.length + const streamResponse: GeminiStreamResponse = { + candidates: [ + { + content: { + parts: [{ text: chunk }], + role: "model", + }, + finishReason: + isLast ? geminiResponse.candidates[0]?.finishReason : undefined, + index: 0, + }, + ], + ...(isLast && geminiResponse.usageMetadata ? + { usageMetadata: geminiResponse.usageMetadata } + : {}), + } + + logGeminiDebug("chunk_sending", endpoint, { + data: { + chunkNumber: Math.floor(i / chunkSize) + 1, + chunk: chunk, + isLast: isLast, + streamResponse: truncateData(streamResponse), + }, + }) + await stream.writeSSE({ data: JSON.stringify(streamResponse) }) + } +} + +// Helper function to send fallback response +async function sendFallbackResponse( + stream: SSEStreamingApi, + geminiResponse: GeminiResponse, + endpoint: string, +) { + logGeminiDebug("fallback_processing", endpoint, { + data: { + candidates: truncateData(geminiResponse.candidates), + context: "Using fallback for non-text or empty content", + }, + }) + const streamResponse: GeminiStreamResponse = { + candidates: geminiResponse.candidates, + usageMetadata: geminiResponse.usageMetadata, + } + + await stream.writeSSE({ data: JSON.stringify(streamResponse) }) +} + +// Helper function to handle streaming response processing +function handleStreamingResponse( + c: Context, + response: AsyncIterable<{ data?: string }>, + endpoint: string, +) { + return streamSSE(c, async (stream) => { + let hasDataSent = false + + try { + for await (const rawEvent of response) { + logGeminiDebug("stream_chunk", endpoint, { data: rawEvent }) + + const shouldContinue = await processStreamChunk( + rawEvent, + endpoint, + stream, + ) + if (!shouldContinue) { + break + } + + if (rawEvent.data && rawEvent.data !== "[DONE]") { + hasDataSent = true + } + } + } catch (streamError) { + await handleStreamError(stream, endpoint, streamError) + } finally { + await ensureCompleteStream(stream, hasDataSent, endpoint) + await stream.close() + } + }) +} + +// Helper function to handle stream errors +async function handleStreamError( + stream: SSEStreamingApi, + endpoint: string, + streamError: unknown, +) { + logGeminiError(endpoint, streamError, { context: "streaming_loop" }) + + try { + await stream.writeSSE({ + data: JSON.stringify({ + error: { + message: "Stream processing error", + type: "internal_error", + }, + }), + }) + } catch (writeError) { + logGeminiError(endpoint, writeError, { + context: "stream_error_write", + }) + } +} + +// Helper function to ensure complete stream +async function ensureCompleteStream( + stream: SSEStreamingApi, + hasDataSent: boolean, + endpoint: string, +) { + if (!hasDataSent) { + try { + await stream.writeSSE({ + data: JSON.stringify({ + candidates: [ + { + content: { parts: [{ text: "" }], role: "model" }, + finishReason: "STOP", + index: 0, + }, + ], + }), + }) + } catch (finalError) { + logGeminiError(endpoint, finalError, { + context: "final_empty_response", + }) + } + } +} + +// Streaming generation endpoint +export async function handleGeminiStreamGeneration(c: Context) { + const endpoint = c.req.url + const model = c.req.param("model") + + logGeminiDebug("handler_entry", endpoint, { + data: { + endpoint: endpoint, + model: model, + context: "Entered handleGeminiStreamGeneration handler", + }, + }) + + try { + await checkRateLimit(state) + + const geminiPayload = await c.req.json() + logGeminiDebug("request", endpoint, { data: geminiPayload }) + + const openAIPayload = translateGeminiToOpenAIStream(geminiPayload, model) + + logGeminiDebug("translation", endpoint, { + data: openAIPayload, + extra: { copilotRequest: openAIPayload }, + }) + + if (state.manualApprove) { + await awaitApproval() + } + + const response = await createChatCompletions(openAIPayload) + + if (isNonStreaming(response)) { + const geminiResponse = translateOpenAIToGemini(response) + logGeminiDebug("response", endpoint, { + data: geminiResponse, + extra: { + copilotResponse: response, + finalResponse: geminiResponse, + }, + }) + + return handleNonStreamingToStreaming(c, geminiResponse, endpoint) + } + + logGeminiDebug("response", endpoint, { + data: "streaming_response_started", + }) + return handleStreamingResponse(c, response, endpoint) + } catch (error) { + logGeminiError(endpoint, error) + const { status, message } = getErrorStatusAndMessage(error) + return c.json({ error: message }, status as 400 | 401 | 404 | 429 | 500) + } +} + +// Token counting endpoint +export async function handleGeminiCountTokens(c: Context) { + const endpoint = c.req.url + const model = c.req.param("model") + + // IMMEDIATE DEBUG: Log that we entered this handler + logGeminiDebug("handler_entry_TOKENS", endpoint, { + data: { + endpoint: endpoint, + model: model, + context: "Entered handleGeminiCountTokens handler", + }, + }) + + try { + const geminiPayload = await c.req.json() + logGeminiDebug("request", endpoint, { data: geminiPayload }) + + const openAIPayload = translateGeminiCountTokensToOpenAI( + geminiPayload, + model, + ) + logGeminiDebug("translation", endpoint, { + data: openAIPayload, + extra: { copilotRequest: openAIPayload }, + }) + + const tokenCounts = getTokenCount(openAIPayload.messages) + logGeminiDebug("token_count", endpoint, { data: tokenCounts }) + + const geminiResponse = translateTokenCountToGemini(tokenCounts.input) + logGeminiDebug("response", endpoint, { + data: geminiResponse, + extra: { finalResponse: geminiResponse }, + }) + + return c.json(geminiResponse) + } catch (error) { + logGeminiError(endpoint, error) + const { status, message } = getErrorStatusAndMessage(error) + return c.json({ error: message }, status as 400 | 401 | 404 | 429 | 500) + } +} + +const isNonStreaming = ( + response: Awaited>, +): response is ChatCompletionResponse => "choices" in response diff --git a/src/routes/messages/gemini-route.ts b/src/routes/messages/gemini-route.ts new file mode 100644 index 000000000..5ee403e37 --- /dev/null +++ b/src/routes/messages/gemini-route.ts @@ -0,0 +1,47 @@ +import { Hono } from "hono" + +import { + handleGeminiGeneration, + handleGeminiStreamGeneration, + handleGeminiCountTokens, +} from "./gemini-handler" + +const router = new Hono() + +// IMPORTANT: Most specific routes FIRST to avoid pattern conflicts +// Use wildcard patterns to handle colons properly + +// Streaming generation endpoint - MOST specific (to avoid conflicts) +// POST /v1beta/{model=models/*}:streamGenerateContent +router.post("/v1beta/models/*", async (c, next) => { + const url = c.req.url + if (url.includes(":streamGenerateContent")) { + return handleGeminiStreamGeneration(c) + } + await next() +}) + +// Token counting endpoint - Second most specific +// POST /v1beta/{model=models/*}:countTokens +router.post("/v1beta/models/*", async (c, next) => { + const url = c.req.url + if (url.includes(":countTokens")) { + return handleGeminiCountTokens(c) + } + await next() +}) + +// Standard generation endpoint - Least specific +// POST /v1beta/{model=models/*}:generateContent +router.post("/v1beta/models/*", async (c, next) => { + const url = c.req.url + if ( + url.includes(":generateContent") + && !url.includes(":streamGenerateContent") + ) { + return handleGeminiGeneration(c) + } + await next() +}) + +export { router as geminiRouter } diff --git a/src/routes/messages/gemini-translation.ts b/src/routes/messages/gemini-translation.ts new file mode 100644 index 000000000..9000573da --- /dev/null +++ b/src/routes/messages/gemini-translation.ts @@ -0,0 +1,439 @@ +import { + type ChatCompletionResponse, + type ChatCompletionChunk, + type ChatCompletionsPayload, + type ContentPart, + type Message, + type Tool, +} from "~/services/copilot/create-chat-completions" + +import { + type GeminiRequest, + type GeminiResponse, + type GeminiContent, + type GeminiPart, + type GeminiTextPart, + type GeminiFunctionCallPart, + type GeminiFunctionResponsePart, + type GeminiTool, + type GeminiCandidate, + type GeminiCountTokensRequest, + type GeminiCountTokensResponse, + type GeminiUsageMetadata, +} from "./gemini-types" + +// Request translation: Gemini -> OpenAI + +export function translateGeminiToOpenAINonStream( + payload: GeminiRequest, + model?: string, +): ChatCompletionsPayload { + return { + model: model || "claude-sonnet-4", // Use provided model or default + messages: translateGeminiContentsToOpenAI( + payload.contents, + payload.systemInstruction, + ), + max_tokens: payload.generationConfig?.maxOutputTokens || 4096, + stop: payload.generationConfig?.stopSequences, + stream: false, + temperature: payload.generationConfig?.temperature, + top_p: payload.generationConfig?.topP, + tools: translateGeminiToolsToOpenAI(payload.tools), + tool_choice: translateGeminiToolConfigToOpenAI(payload.toolConfig), + } +} + +export function translateGeminiToOpenAIStream( + payload: GeminiRequest, + model?: string, +): ChatCompletionsPayload { + const result = { + model: model || "claude-sonnet-4", // Use provided model or default + messages: translateGeminiContentsToOpenAI( + payload.contents, + payload.systemInstruction, + ), + max_tokens: payload.generationConfig?.maxOutputTokens || 4096, + stop: payload.generationConfig?.stopSequences, + stream: true, + temperature: payload.generationConfig?.temperature, + top_p: payload.generationConfig?.topP, + tools: translateGeminiToolsToOpenAI(payload.tools), + tool_choice: translateGeminiToolConfigToOpenAI(payload.toolConfig), + } + + return result +} + +function translateGeminiContentsToOpenAI( + contents: Array, + systemInstruction?: GeminiContent, +): Array { + const messages: Array = [] + + // Add system instruction first if present + if (systemInstruction) { + const systemText = extractTextFromGeminiContent(systemInstruction) + if (systemText) { + messages.push({ role: "system", content: systemText }) + } + } + + // Process conversation contents + for (const content of contents) { + const role = content.role === "model" ? "assistant" : "user" + + // Check for function calls/responses + const functionCalls = content.parts.filter( + (part): part is GeminiFunctionCallPart => "functionCall" in part, + ) + const functionResponses = content.parts.filter( + (part): part is GeminiFunctionResponsePart => "functionResponse" in part, + ) + + if (functionResponses.length > 0) { + // Add tool result messages + for (const funcResponse of functionResponses) { + messages.push({ + role: "tool", + tool_call_id: generateToolCallId(funcResponse.functionResponse.name), + content: JSON.stringify(funcResponse.functionResponse.response), + }) + } + } + + if (functionCalls.length > 0 && role === "assistant") { + // Assistant message with tool calls + const textContent = extractTextFromGeminiContent(content) + messages.push({ + role: "assistant", + content: textContent || null, + tool_calls: functionCalls.map((call) => ({ + id: generateToolCallId(call.functionCall.name), + type: "function", + function: { + name: call.functionCall.name, + arguments: JSON.stringify(call.functionCall.args), + }, + })), + }) + } else { + // Regular message + const messageContent = translateGeminiContentToOpenAI(content) + if (messageContent) { + messages.push({ role, content: messageContent }) + } + } + } + + return messages +} + +function translateGeminiContentToOpenAI( + content: GeminiContent, +): string | Array | null { + if (content.parts.length === 0) return null + + const hasMedia = content.parts.some((part) => "inlineData" in part) + + if (!hasMedia) { + // Text-only content + return extractTextFromGeminiContent(content) + } + + // Mixed content with media + const contentParts: Array = [] + for (const part of content.parts) { + if ("text" in part) { + contentParts.push({ type: "text", text: part.text }) + } else if ("inlineData" in part) { + contentParts.push({ + type: "image_url", + image_url: { + url: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`, + }, + }) + } + } + + return contentParts +} + +function extractTextFromGeminiContent(content: GeminiContent): string { + return content.parts + .filter((part): part is GeminiTextPart => "text" in part) + .map((part) => part.text) + .join("\n\n") +} + +function translateGeminiToolsToOpenAI( + geminiTools?: Array, +): Array | undefined { + if (!geminiTools || geminiTools.length === 0) return undefined + + const tools: Array = [] + for (const tool of geminiTools) { + for (const func of tool.functionDeclarations) { + tools.push({ + type: "function", + function: { + name: func.name, + description: func.description, + parameters: func.parameters, + }, + }) + } + } + + return tools +} + +function translateGeminiToolConfigToOpenAI( + toolConfig?: GeminiRequest["toolConfig"], +): ChatCompletionsPayload["tool_choice"] { + if (!toolConfig) return undefined + + const mode = toolConfig.functionCallingConfig.mode + switch (mode) { + case "AUTO": { + return "auto" + } + case "ANY": { + return "required" + } + case "NONE": { + return "none" + } + default: { + return undefined + } + } +} + +// Response translation: OpenAI -> Gemini + +export function translateOpenAIToGemini( + response: ChatCompletionResponse, +): GeminiResponse { + const candidates: Array = response.choices.map( + (choice, index) => ({ + content: translateOpenAIMessageToGeminiContent(choice.message), + finishReason: mapOpenAIFinishReasonToGemini(choice.finish_reason), + index, + }), + ) + + return { + candidates, + usageMetadata: { + promptTokenCount: response.usage?.prompt_tokens || 0, + candidatesTokenCount: response.usage?.completion_tokens || 0, + totalTokenCount: response.usage?.total_tokens || 0, + }, + } +} + +function translateOpenAIMessageToGeminiContent( + message: Message, +): GeminiContent { + const parts: Array = [] + + // Handle text content + if (typeof message.content === "string") { + if (message.content) { + parts.push({ text: message.content }) + } + } else if (Array.isArray(message.content)) { + for (const part of message.content) { + if (part.type === "text") { + parts.push({ text: part.text }) + } else { + // Convert data URL back to inline data + const match = part.image_url.url.match(/^data:([^;]+);base64,(.+)$/) + if (match) { + parts.push({ + inlineData: { + mimeType: match[1], + data: match[2], + }, + }) + } + } + } + } + + // Handle tool calls + if (message.tool_calls) { + for (const toolCall of message.tool_calls) { + parts.push({ + functionCall: { + name: toolCall.function.name, + args: + toolCall.function.arguments ? + (JSON.parse(toolCall.function.arguments) as Record< + string, + unknown + >) + : {}, + }, + }) + } + } + + return { + parts, + role: "model", + } +} + +function mapOpenAIFinishReasonToGemini( + finishReason: string | null, +): GeminiCandidate["finishReason"] { + switch (finishReason) { + case "stop": { + return "STOP" + } + case "length": { + return "MAX_TOKENS" + } + case "content_filter": { + return "SAFETY" + } + case "tool_calls": { + return "STOP" + } // Gemini doesn't have a specific tool_calls finish reason + default: { + return "FINISH_REASON_UNSPECIFIED" + } + } +} + +// Utility functions + +function generateToolCallId(functionName: string): string { + return `call_${functionName}_${Date.now()}_${Math.random().toString(36).slice(2, 11)}` +} + +// Helper function to process tool calls in streaming chunks +function processToolCalls( + toolCalls: Array<{ + index: number + id?: string + type?: "function" + function?: { + name?: string + arguments?: string + } + }>, +): Array { + const parts: Array = [] + + for (const toolCall of toolCalls) { + if (!toolCall.function?.name) { + continue + } + + let args: Record + try { + args = JSON.parse(toolCall.function.arguments || "{}") as Record< + string, + unknown + > + } catch { + // In streaming, arguments might be incomplete JSON + // Skip this chunk and wait for complete arguments + continue + } + + parts.push({ + functionCall: { + name: toolCall.function.name, + args, + }, + }) + } + + return parts +} + +// Helper function to create usage metadata +function createUsageMetadata(chunk: ChatCompletionChunk): GeminiUsageMetadata { + return { + promptTokenCount: chunk.usage?.prompt_tokens || 0, + candidatesTokenCount: chunk.usage?.completion_tokens || 0, + totalTokenCount: chunk.usage?.total_tokens || 0, + } +} + +// Stream translation: OpenAI Chunk -> Gemini Stream Response +export function translateOpenAIChunkToGemini(chunk: ChatCompletionChunk): { + candidates: Array + usageMetadata?: GeminiUsageMetadata +} | null { + if (chunk.choices.length === 0) { + return null + } + + const choice = chunk.choices[0] + const parts: Array = [] + + if (choice.delta.content) { + parts.push({ text: choice.delta.content }) + } + + if (choice.delta.tool_calls) { + parts.push(...processToolCalls(choice.delta.tool_calls)) + } + + if (parts.length === 0 && !choice.finish_reason) { + return null + } + + const candidate: GeminiCandidate = { + content: { + parts, + role: "model", + }, + finishReason: mapOpenAIFinishReasonToGemini(choice.finish_reason), + index: choice.index, + } + + const response: { + candidates: Array + usageMetadata?: GeminiUsageMetadata + } = { + candidates: [candidate], + } + + if (choice.finish_reason) { + response.usageMetadata = createUsageMetadata(chunk) + } + + return response +} + +// Token counting translation + +export function translateGeminiCountTokensToOpenAI( + request: GeminiCountTokensRequest, + model?: string, +): ChatCompletionsPayload { + return { + model: model || "claude-sonnet-4", + messages: translateGeminiContentsToOpenAI( + request.contents, + request.systemInstruction, + ), + max_tokens: 1, // Minimal for token counting + tools: translateGeminiToolsToOpenAI(request.tools), + } +} + +export function translateTokenCountToGemini( + totalTokens: number, +): GeminiCountTokensResponse { + return { + totalTokens, + } +} diff --git a/src/routes/messages/gemini-types.ts b/src/routes/messages/gemini-types.ts new file mode 100644 index 000000000..98ba5aff8 --- /dev/null +++ b/src/routes/messages/gemini-types.ts @@ -0,0 +1,123 @@ +// Gemini API Types + +export interface GeminiRequest { + contents: Array + tools?: Array + toolConfig?: GeminiToolConfig + safetySettings?: Array + systemInstruction?: GeminiContent + generationConfig?: GeminiGenerationConfig +} + +export interface GeminiContent { + parts: Array + role?: "user" | "model" +} + +export type GeminiPart = + | GeminiTextPart + | GeminiInlineDataPart + | GeminiFunctionCallPart + | GeminiFunctionResponsePart + +export interface GeminiTextPart { + text: string +} + +export interface GeminiInlineDataPart { + inlineData: { + mimeType: string + data: string + } +} + +export interface GeminiFunctionCallPart { + functionCall: { + name: string + args: Record + } +} + +export interface GeminiFunctionResponsePart { + functionResponse: { + name: string + response: Record + } +} + +export interface GeminiTool { + functionDeclarations: Array +} + +export interface GeminiFunctionDeclaration { + name: string + description?: string + parameters: Record +} + +export interface GeminiToolConfig { + functionCallingConfig: { + mode: "AUTO" | "ANY" | "NONE" + allowedFunctionNames?: Array + } +} + +export interface GeminiSafetySetting { + category: string + threshold: string +} + +export interface GeminiGenerationConfig { + stopSequences?: Array + temperature?: number + maxOutputTokens?: number + topP?: number + topK?: number +} + +// Response types +export interface GeminiResponse { + candidates: Array + usageMetadata?: GeminiUsageMetadata +} + +export interface GeminiCandidate { + content: GeminiContent + finishReason?: + | "FINISH_REASON_UNSPECIFIED" + | "STOP" + | "MAX_TOKENS" + | "SAFETY" + | "RECITATION" + | "OTHER" + index: number + safetyRatings?: Array +} + +export interface GeminiSafetyRating { + category: string + probability: string +} + +export interface GeminiUsageMetadata { + promptTokenCount: number + candidatesTokenCount: number + totalTokenCount: number +} + +// Token counting types +export interface GeminiCountTokensRequest { + contents: Array + tools?: Array + systemInstruction?: GeminiContent +} + +export interface GeminiCountTokensResponse { + totalTokens: number +} + +// Streaming types +export interface GeminiStreamResponse { + candidates?: Array + usageMetadata?: GeminiUsageMetadata +} From edd28750e17c3e3ff91c103e861611055ed0f0c8 Mon Sep 17 00:00:00 2001 From: cpf Date: Sun, 21 Sep 2025 23:43:31 +0800 Subject: [PATCH 03/16] feat: route in server --- src/server.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/server.ts b/src/server.ts index 3cb2bb860..0b95ef8c2 100644 --- a/src/server.ts +++ b/src/server.ts @@ -4,6 +4,7 @@ import { logger } from "hono/logger" import { completionRoutes } from "./routes/chat-completions/route" import { embeddingRoutes } from "./routes/embeddings/route" +import { geminiRouter } from "./routes/generate-content/route" import { messageRoutes } from "./routes/messages/route" import { modelRoutes } from "./routes/models/route" import { tokenRoute } from "./routes/token/route" @@ -26,6 +27,7 @@ server.route("/token", tokenRoute) server.route("/v1/chat/completions", completionRoutes) server.route("/v1/models", modelRoutes) server.route("/v1/embeddings", embeddingRoutes) +server.route("/", geminiRouter) // Anthropic compatible endpoints server.route("/v1/messages", messageRoutes) From bf5188ea150740998e1693d8e2c6bc7740e2d1c8 Mon Sep 17 00:00:00 2001 From: cpf Date: Sun, 21 Sep 2025 23:48:24 +0800 Subject: [PATCH 04/16] feat: add idle timeout configuration for server --- src/start.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/start.ts b/src/start.ts index a1b02303e..255285a43 100644 --- a/src/start.ts +++ b/src/start.ts @@ -103,10 +103,12 @@ export async function runServer(options: RunServerOptions): Promise { consola.box( `🌐 Usage Viewer: https://ericc-ch.github.io/copilot-api?endpoint=${serverUrl}/usage`, ) - serve({ fetch: server.fetch as ServerHandler, port: options.port, + bun: { + idleTimeout: 255, // gemini timeout + }, }) } From e6af851cb64d2148a90fb74ce6bc181caaf0bc15 Mon Sep 17 00:00:00 2001 From: cpf Date: Sun, 21 Sep 2025 23:52:55 +0800 Subject: [PATCH 05/16] Revert "lint" This reverts commit fa6b6124648908e91741e7694a3e2420ebf140b4. --- src/routes/messages/gemini-handler.ts | 627 ---------------------- src/routes/messages/gemini-route.ts | 47 -- src/routes/messages/gemini-translation.ts | 439 --------------- src/routes/messages/gemini-types.ts | 123 ----- 4 files changed, 1236 deletions(-) delete mode 100644 src/routes/messages/gemini-handler.ts delete mode 100644 src/routes/messages/gemini-route.ts delete mode 100644 src/routes/messages/gemini-translation.ts delete mode 100644 src/routes/messages/gemini-types.ts diff --git a/src/routes/messages/gemini-handler.ts b/src/routes/messages/gemini-handler.ts deleted file mode 100644 index b3db02409..000000000 --- a/src/routes/messages/gemini-handler.ts +++ /dev/null @@ -1,627 +0,0 @@ -import type { Context } from "hono" -import type { SSEStreamingApi } from "hono/streaming" - -import consola from "consola" -import { streamSSE } from "hono/streaming" -import { promises as fs } from "node:fs" -import path from "node:path" - -import { awaitApproval } from "~/lib/approval" -import { checkRateLimit } from "~/lib/rate-limit" -import { state } from "~/lib/state" -import { getTokenCount } from "~/lib/tokenizer" -import { - createChatCompletions, - type ChatCompletionChunk, - type ChatCompletionResponse, -} from "~/services/copilot/create-chat-completions" - -import { - translateGeminiToOpenAINonStream, - translateGeminiToOpenAIStream, - translateOpenAIToGemini, - translateGeminiCountTokensToOpenAI, - translateTokenCountToGemini, - translateOpenAIChunkToGemini, -} from "./gemini-translation" -import { - type GeminiRequest, - type GeminiCountTokensRequest, - type GeminiStreamResponse, - type GeminiResponse, -} from "./gemini-types" - -// Debug logging interface -interface GeminiDebugLog { - timestamp: string - type: - | "request" - | "response" - | "translation" - | "error" - | "stream_chunk" - | "stream_translation" - endpoint: string - data: unknown - copilotRequest?: unknown - copilotResponse?: unknown - finalResponse?: unknown -} - -// File logging functions -async function writeLogToFile(logEntry: GeminiDebugLog) { - const logsDir = path.join(process.cwd(), "logs") - - try { - // Ensure logs directory exists - await fs.mkdir(logsDir, { recursive: true }) - - const logLine = JSON.stringify(logEntry) + "\n" - - // Write to main debug log - await fs.appendFile(path.join(logsDir, "gemini-debug.log"), logLine) - - // Write to specific logs based on type - if (logEntry.type === "error") { - await fs.appendFile(path.join(logsDir, "gemini-errors.log"), logLine) - } else if ( - logEntry.type === "translation" - || logEntry.type === "stream_translation" - ) { - await fs.appendFile(path.join(logsDir, "gemini-translation.log"), logLine) - } - } catch (error) { - consola.error("Failed to write log file:", error) - } -} - -// Helper function to truncate data for logging -function truncateData(data: unknown, maxLength = 200): unknown { - if (typeof data === "string") { - return data.length > maxLength ? `${data.slice(0, maxLength)}...` : data - } - - if (Array.isArray(data)) { - return data.map((item) => truncateData(item, maxLength)) - } - - if (data && typeof data === "object") { - const obj = data as Record - const result: Record = {} - - for (const [key, value] of Object.entries(obj)) { - if (key === "messages" && Array.isArray(value)) { - result[key] = value.map((msg: { role: string; content: unknown }) => ({ - role: msg.role, - content: getContentDisplay(msg.content), - })) - } else if (key === "contents" && Array.isArray(value)) { - result[key] = value.map( - (content: { role: string; parts?: Array }) => ({ - role: content.role, - parts: - Array.isArray(content.parts) && content.parts.length > 0 ? - `[${content.parts.length} parts]` - : content.parts, - }), - ) - } else { - result[key] = truncateData(value, maxLength) - } - } - return result - } - - return data -} - -// Helper function to display content for logging -function getContentDisplay(content: unknown): unknown { - if (typeof content === "string") { - return truncateData(content, 100) - } - if (Array.isArray(content) && content.length > 0) { - return `[content array: ${content.length} items]` - } - return content -} - -// Debug logging functions -function logGeminiDebug( - type: string, - endpoint: string, - options: { data: unknown; extra?: Record }, -) { - const { data, extra } = options - const truncatedData = truncateData(data) - const truncatedExtra = extra ? truncateData(extra) : undefined - - const logEntry: GeminiDebugLog = { - timestamp: new Date().toISOString(), - type: type as GeminiDebugLog["type"], - endpoint, - data: truncatedData, - ...(truncatedExtra as Record), - } - - // Console logging - more concise - const endpointPath = new URL(endpoint).pathname - consola.debug(`[GEMINI-${type.toUpperCase()}] ${endpointPath}`) - - // File logging (async, don't wait) - now always write but with truncated data - writeLogToFile(logEntry).catch((error: unknown) => - consola.error("Log file write error:", error), - ) -} - -function logGeminiError(endpoint: string, error: unknown, data?: unknown) { - const truncatedData = data ? truncateData(data) : undefined - - const logEntry: GeminiDebugLog = { - timestamp: new Date().toISOString(), - type: "error", - endpoint, - data: { - error: error instanceof Error ? error.message : String(error), - stack: error instanceof Error ? error.stack : undefined, - data: truncatedData, - }, - } - - // Console logging - more concise - const endpointPath = new URL(endpoint).pathname - consola.error( - `[GEMINI-ERROR] ${endpointPath}: ${error instanceof Error ? error.message : String(error)}`, - ) - - // File logging (async, don't wait) - writeLogToFile(logEntry).catch((logError: unknown) => - consola.error("Log file write error:", logError), - ) -} - -// Helper function to process stream chunk -async function processStreamChunk( - rawEvent: { data?: string }, - endpoint: string, - stream: SSEStreamingApi, -): Promise { - if (rawEvent.data === "[DONE]") { - return false // Signal to stop processing - } - - if (!rawEvent.data) { - return true // Continue processing - } - - try { - const chunkData = JSON.parse(rawEvent.data) as unknown - const chunk = chunkData as ChatCompletionChunk - const geminiResponse = translateOpenAIChunkToGemini(chunk) - - if (geminiResponse) { - consola.debug("Streaming geminiResponse object:", geminiResponse) - const jsonLine = JSON.stringify(geminiResponse) - consola.debug("Streaming JSON line:", jsonLine) - consola.debug("About to send SSE data:", jsonLine.slice(0, 100)) - - // Validate JSON before sending - try { - JSON.parse(jsonLine) - } catch (validateError) { - logGeminiError(endpoint, validateError, { - rawEvent, - context: "JSON validation failed before sending", - jsonLine: jsonLine.slice(0, 200), - }) - return true // Continue processing - } - - await stream.writeSSE({ - data: jsonLine, - }) - return true // Continue processing - } - return true // Continue processing - } catch (chunkError) { - logGeminiError(endpoint, chunkError, { - rawEvent, - context: "JSON.parse failed in stream", - }) - return true // Continue processing - } -} - -// Error handling helper -function getErrorStatusAndMessage(error: unknown): { - status: number - message: string -} { - if (!(error instanceof Error)) { - return { status: 500, message: "Internal server error" } - } - - const errorMappings = [ - { - condition: (err: Error) => - err.name === "RateLimitError" || err.message.includes("rate limit"), - status: 429, - message: "Rate limit exceeded", - }, - { - condition: (err: Error) => - err.name === "ValidationError" || err.message.includes("validation"), - status: 400, - message: "Invalid request", - }, - { - condition: (err: Error) => - err.name === "AuthenticationError" || err.message.includes("auth"), - status: 401, - message: "Authentication failed", - }, - { - condition: (err: Error) => - err.name === "NotFoundError" || err.message.includes("not found"), - status: 404, - message: "Resource not found", - }, - ] - - for (const mapping of errorMappings) { - if (mapping.condition(error)) { - return { status: mapping.status, message: mapping.message } - } - } - - return { status: 500, message: "Internal server error" } -} - -// Standard generation endpoint -export async function handleGeminiGeneration(c: Context) { - const endpoint = c.req.url - const model = c.req.param("model") - - // IMMEDIATE DEBUG: Log that we entered this handler - logGeminiDebug("handler_entry_GENERATION", endpoint, { - data: { - endpoint: endpoint, - model: model, - context: "Entered handleGeminiGeneration handler (NON-STREAMING)", - }, - }) - - try { - await checkRateLimit(state) - - const geminiPayload = await c.req.json() - logGeminiDebug("request", endpoint, { data: geminiPayload }) - - const openAIPayload = translateGeminiToOpenAINonStream(geminiPayload, model) - logGeminiDebug("translation", endpoint, { - data: openAIPayload, - extra: { copilotRequest: openAIPayload }, - }) - - if (state.manualApprove) { - await awaitApproval() - } - - const response = await createChatCompletions(openAIPayload) - - if (isNonStreaming(response)) { - logGeminiDebug("response", endpoint, { - data: response, - extra: { copilotResponse: response }, - }) - - const geminiResponse = translateOpenAIToGemini(response) - logGeminiDebug("translation", endpoint, { - data: geminiResponse, - extra: { finalResponse: geminiResponse }, - }) - - return c.json(geminiResponse) - } - - // This shouldn't happen for non-streaming endpoint - logGeminiError( - endpoint, - new Error("Unexpected streaming response for non-streaming endpoint"), - ) - return c.json({ error: "Internal error" }, 500) - } catch (error) { - logGeminiError(endpoint, error) - const { status, message } = getErrorStatusAndMessage(error) - return c.json({ error: message }, status as 400 | 401 | 404 | 429 | 500) - } -} - -// Helper function to handle non-streaming response conversion -function handleNonStreamingToStreaming( - c: Context, - geminiResponse: GeminiResponse, - endpoint: string, -) { - return streamSSE(c, async (stream) => { - logGeminiDebug("non_streaming_conversion", endpoint, { - data: { - geminiResponse: truncateData(geminiResponse), - context: "Converting non-streaming response to streaming", - }, - }) - - const textContent = geminiResponse.candidates[0]?.content?.parts?.[0] - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - await (textContent && "text" in textContent ? - sendTextInChunks(stream, textContent.text, { - geminiResponse, - endpoint, - }) - : sendFallbackResponse(stream, geminiResponse, endpoint)) - - logGeminiDebug("stream_closing", endpoint, { - data: { context: "Closing non-streaming to streaming conversion" }, - }) - await stream.close() - }) -} - -// Helper function to send text in chunks with configuration object -async function sendTextInChunks( - stream: SSEStreamingApi, - text: string, - options: { geminiResponse: GeminiResponse; endpoint: string }, -) { - const { geminiResponse, endpoint } = options - logGeminiDebug("text_chunking", endpoint, { - data: { - text: text, - textLength: text.length, - context: "Processing text for chunking", - }, - }) - const chunkSize = Math.max(1, Math.min(50, text.length)) - - for (let i = 0; i < text.length; i += chunkSize) { - const chunk = text.slice(i, i + chunkSize) - const isLast = i + chunkSize >= text.length - const streamResponse: GeminiStreamResponse = { - candidates: [ - { - content: { - parts: [{ text: chunk }], - role: "model", - }, - finishReason: - isLast ? geminiResponse.candidates[0]?.finishReason : undefined, - index: 0, - }, - ], - ...(isLast && geminiResponse.usageMetadata ? - { usageMetadata: geminiResponse.usageMetadata } - : {}), - } - - logGeminiDebug("chunk_sending", endpoint, { - data: { - chunkNumber: Math.floor(i / chunkSize) + 1, - chunk: chunk, - isLast: isLast, - streamResponse: truncateData(streamResponse), - }, - }) - await stream.writeSSE({ data: JSON.stringify(streamResponse) }) - } -} - -// Helper function to send fallback response -async function sendFallbackResponse( - stream: SSEStreamingApi, - geminiResponse: GeminiResponse, - endpoint: string, -) { - logGeminiDebug("fallback_processing", endpoint, { - data: { - candidates: truncateData(geminiResponse.candidates), - context: "Using fallback for non-text or empty content", - }, - }) - const streamResponse: GeminiStreamResponse = { - candidates: geminiResponse.candidates, - usageMetadata: geminiResponse.usageMetadata, - } - - await stream.writeSSE({ data: JSON.stringify(streamResponse) }) -} - -// Helper function to handle streaming response processing -function handleStreamingResponse( - c: Context, - response: AsyncIterable<{ data?: string }>, - endpoint: string, -) { - return streamSSE(c, async (stream) => { - let hasDataSent = false - - try { - for await (const rawEvent of response) { - logGeminiDebug("stream_chunk", endpoint, { data: rawEvent }) - - const shouldContinue = await processStreamChunk( - rawEvent, - endpoint, - stream, - ) - if (!shouldContinue) { - break - } - - if (rawEvent.data && rawEvent.data !== "[DONE]") { - hasDataSent = true - } - } - } catch (streamError) { - await handleStreamError(stream, endpoint, streamError) - } finally { - await ensureCompleteStream(stream, hasDataSent, endpoint) - await stream.close() - } - }) -} - -// Helper function to handle stream errors -async function handleStreamError( - stream: SSEStreamingApi, - endpoint: string, - streamError: unknown, -) { - logGeminiError(endpoint, streamError, { context: "streaming_loop" }) - - try { - await stream.writeSSE({ - data: JSON.stringify({ - error: { - message: "Stream processing error", - type: "internal_error", - }, - }), - }) - } catch (writeError) { - logGeminiError(endpoint, writeError, { - context: "stream_error_write", - }) - } -} - -// Helper function to ensure complete stream -async function ensureCompleteStream( - stream: SSEStreamingApi, - hasDataSent: boolean, - endpoint: string, -) { - if (!hasDataSent) { - try { - await stream.writeSSE({ - data: JSON.stringify({ - candidates: [ - { - content: { parts: [{ text: "" }], role: "model" }, - finishReason: "STOP", - index: 0, - }, - ], - }), - }) - } catch (finalError) { - logGeminiError(endpoint, finalError, { - context: "final_empty_response", - }) - } - } -} - -// Streaming generation endpoint -export async function handleGeminiStreamGeneration(c: Context) { - const endpoint = c.req.url - const model = c.req.param("model") - - logGeminiDebug("handler_entry", endpoint, { - data: { - endpoint: endpoint, - model: model, - context: "Entered handleGeminiStreamGeneration handler", - }, - }) - - try { - await checkRateLimit(state) - - const geminiPayload = await c.req.json() - logGeminiDebug("request", endpoint, { data: geminiPayload }) - - const openAIPayload = translateGeminiToOpenAIStream(geminiPayload, model) - - logGeminiDebug("translation", endpoint, { - data: openAIPayload, - extra: { copilotRequest: openAIPayload }, - }) - - if (state.manualApprove) { - await awaitApproval() - } - - const response = await createChatCompletions(openAIPayload) - - if (isNonStreaming(response)) { - const geminiResponse = translateOpenAIToGemini(response) - logGeminiDebug("response", endpoint, { - data: geminiResponse, - extra: { - copilotResponse: response, - finalResponse: geminiResponse, - }, - }) - - return handleNonStreamingToStreaming(c, geminiResponse, endpoint) - } - - logGeminiDebug("response", endpoint, { - data: "streaming_response_started", - }) - return handleStreamingResponse(c, response, endpoint) - } catch (error) { - logGeminiError(endpoint, error) - const { status, message } = getErrorStatusAndMessage(error) - return c.json({ error: message }, status as 400 | 401 | 404 | 429 | 500) - } -} - -// Token counting endpoint -export async function handleGeminiCountTokens(c: Context) { - const endpoint = c.req.url - const model = c.req.param("model") - - // IMMEDIATE DEBUG: Log that we entered this handler - logGeminiDebug("handler_entry_TOKENS", endpoint, { - data: { - endpoint: endpoint, - model: model, - context: "Entered handleGeminiCountTokens handler", - }, - }) - - try { - const geminiPayload = await c.req.json() - logGeminiDebug("request", endpoint, { data: geminiPayload }) - - const openAIPayload = translateGeminiCountTokensToOpenAI( - geminiPayload, - model, - ) - logGeminiDebug("translation", endpoint, { - data: openAIPayload, - extra: { copilotRequest: openAIPayload }, - }) - - const tokenCounts = getTokenCount(openAIPayload.messages) - logGeminiDebug("token_count", endpoint, { data: tokenCounts }) - - const geminiResponse = translateTokenCountToGemini(tokenCounts.input) - logGeminiDebug("response", endpoint, { - data: geminiResponse, - extra: { finalResponse: geminiResponse }, - }) - - return c.json(geminiResponse) - } catch (error) { - logGeminiError(endpoint, error) - const { status, message } = getErrorStatusAndMessage(error) - return c.json({ error: message }, status as 400 | 401 | 404 | 429 | 500) - } -} - -const isNonStreaming = ( - response: Awaited>, -): response is ChatCompletionResponse => "choices" in response diff --git a/src/routes/messages/gemini-route.ts b/src/routes/messages/gemini-route.ts deleted file mode 100644 index 5ee403e37..000000000 --- a/src/routes/messages/gemini-route.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { Hono } from "hono" - -import { - handleGeminiGeneration, - handleGeminiStreamGeneration, - handleGeminiCountTokens, -} from "./gemini-handler" - -const router = new Hono() - -// IMPORTANT: Most specific routes FIRST to avoid pattern conflicts -// Use wildcard patterns to handle colons properly - -// Streaming generation endpoint - MOST specific (to avoid conflicts) -// POST /v1beta/{model=models/*}:streamGenerateContent -router.post("/v1beta/models/*", async (c, next) => { - const url = c.req.url - if (url.includes(":streamGenerateContent")) { - return handleGeminiStreamGeneration(c) - } - await next() -}) - -// Token counting endpoint - Second most specific -// POST /v1beta/{model=models/*}:countTokens -router.post("/v1beta/models/*", async (c, next) => { - const url = c.req.url - if (url.includes(":countTokens")) { - return handleGeminiCountTokens(c) - } - await next() -}) - -// Standard generation endpoint - Least specific -// POST /v1beta/{model=models/*}:generateContent -router.post("/v1beta/models/*", async (c, next) => { - const url = c.req.url - if ( - url.includes(":generateContent") - && !url.includes(":streamGenerateContent") - ) { - return handleGeminiGeneration(c) - } - await next() -}) - -export { router as geminiRouter } diff --git a/src/routes/messages/gemini-translation.ts b/src/routes/messages/gemini-translation.ts deleted file mode 100644 index 9000573da..000000000 --- a/src/routes/messages/gemini-translation.ts +++ /dev/null @@ -1,439 +0,0 @@ -import { - type ChatCompletionResponse, - type ChatCompletionChunk, - type ChatCompletionsPayload, - type ContentPart, - type Message, - type Tool, -} from "~/services/copilot/create-chat-completions" - -import { - type GeminiRequest, - type GeminiResponse, - type GeminiContent, - type GeminiPart, - type GeminiTextPart, - type GeminiFunctionCallPart, - type GeminiFunctionResponsePart, - type GeminiTool, - type GeminiCandidate, - type GeminiCountTokensRequest, - type GeminiCountTokensResponse, - type GeminiUsageMetadata, -} from "./gemini-types" - -// Request translation: Gemini -> OpenAI - -export function translateGeminiToOpenAINonStream( - payload: GeminiRequest, - model?: string, -): ChatCompletionsPayload { - return { - model: model || "claude-sonnet-4", // Use provided model or default - messages: translateGeminiContentsToOpenAI( - payload.contents, - payload.systemInstruction, - ), - max_tokens: payload.generationConfig?.maxOutputTokens || 4096, - stop: payload.generationConfig?.stopSequences, - stream: false, - temperature: payload.generationConfig?.temperature, - top_p: payload.generationConfig?.topP, - tools: translateGeminiToolsToOpenAI(payload.tools), - tool_choice: translateGeminiToolConfigToOpenAI(payload.toolConfig), - } -} - -export function translateGeminiToOpenAIStream( - payload: GeminiRequest, - model?: string, -): ChatCompletionsPayload { - const result = { - model: model || "claude-sonnet-4", // Use provided model or default - messages: translateGeminiContentsToOpenAI( - payload.contents, - payload.systemInstruction, - ), - max_tokens: payload.generationConfig?.maxOutputTokens || 4096, - stop: payload.generationConfig?.stopSequences, - stream: true, - temperature: payload.generationConfig?.temperature, - top_p: payload.generationConfig?.topP, - tools: translateGeminiToolsToOpenAI(payload.tools), - tool_choice: translateGeminiToolConfigToOpenAI(payload.toolConfig), - } - - return result -} - -function translateGeminiContentsToOpenAI( - contents: Array, - systemInstruction?: GeminiContent, -): Array { - const messages: Array = [] - - // Add system instruction first if present - if (systemInstruction) { - const systemText = extractTextFromGeminiContent(systemInstruction) - if (systemText) { - messages.push({ role: "system", content: systemText }) - } - } - - // Process conversation contents - for (const content of contents) { - const role = content.role === "model" ? "assistant" : "user" - - // Check for function calls/responses - const functionCalls = content.parts.filter( - (part): part is GeminiFunctionCallPart => "functionCall" in part, - ) - const functionResponses = content.parts.filter( - (part): part is GeminiFunctionResponsePart => "functionResponse" in part, - ) - - if (functionResponses.length > 0) { - // Add tool result messages - for (const funcResponse of functionResponses) { - messages.push({ - role: "tool", - tool_call_id: generateToolCallId(funcResponse.functionResponse.name), - content: JSON.stringify(funcResponse.functionResponse.response), - }) - } - } - - if (functionCalls.length > 0 && role === "assistant") { - // Assistant message with tool calls - const textContent = extractTextFromGeminiContent(content) - messages.push({ - role: "assistant", - content: textContent || null, - tool_calls: functionCalls.map((call) => ({ - id: generateToolCallId(call.functionCall.name), - type: "function", - function: { - name: call.functionCall.name, - arguments: JSON.stringify(call.functionCall.args), - }, - })), - }) - } else { - // Regular message - const messageContent = translateGeminiContentToOpenAI(content) - if (messageContent) { - messages.push({ role, content: messageContent }) - } - } - } - - return messages -} - -function translateGeminiContentToOpenAI( - content: GeminiContent, -): string | Array | null { - if (content.parts.length === 0) return null - - const hasMedia = content.parts.some((part) => "inlineData" in part) - - if (!hasMedia) { - // Text-only content - return extractTextFromGeminiContent(content) - } - - // Mixed content with media - const contentParts: Array = [] - for (const part of content.parts) { - if ("text" in part) { - contentParts.push({ type: "text", text: part.text }) - } else if ("inlineData" in part) { - contentParts.push({ - type: "image_url", - image_url: { - url: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`, - }, - }) - } - } - - return contentParts -} - -function extractTextFromGeminiContent(content: GeminiContent): string { - return content.parts - .filter((part): part is GeminiTextPart => "text" in part) - .map((part) => part.text) - .join("\n\n") -} - -function translateGeminiToolsToOpenAI( - geminiTools?: Array, -): Array | undefined { - if (!geminiTools || geminiTools.length === 0) return undefined - - const tools: Array = [] - for (const tool of geminiTools) { - for (const func of tool.functionDeclarations) { - tools.push({ - type: "function", - function: { - name: func.name, - description: func.description, - parameters: func.parameters, - }, - }) - } - } - - return tools -} - -function translateGeminiToolConfigToOpenAI( - toolConfig?: GeminiRequest["toolConfig"], -): ChatCompletionsPayload["tool_choice"] { - if (!toolConfig) return undefined - - const mode = toolConfig.functionCallingConfig.mode - switch (mode) { - case "AUTO": { - return "auto" - } - case "ANY": { - return "required" - } - case "NONE": { - return "none" - } - default: { - return undefined - } - } -} - -// Response translation: OpenAI -> Gemini - -export function translateOpenAIToGemini( - response: ChatCompletionResponse, -): GeminiResponse { - const candidates: Array = response.choices.map( - (choice, index) => ({ - content: translateOpenAIMessageToGeminiContent(choice.message), - finishReason: mapOpenAIFinishReasonToGemini(choice.finish_reason), - index, - }), - ) - - return { - candidates, - usageMetadata: { - promptTokenCount: response.usage?.prompt_tokens || 0, - candidatesTokenCount: response.usage?.completion_tokens || 0, - totalTokenCount: response.usage?.total_tokens || 0, - }, - } -} - -function translateOpenAIMessageToGeminiContent( - message: Message, -): GeminiContent { - const parts: Array = [] - - // Handle text content - if (typeof message.content === "string") { - if (message.content) { - parts.push({ text: message.content }) - } - } else if (Array.isArray(message.content)) { - for (const part of message.content) { - if (part.type === "text") { - parts.push({ text: part.text }) - } else { - // Convert data URL back to inline data - const match = part.image_url.url.match(/^data:([^;]+);base64,(.+)$/) - if (match) { - parts.push({ - inlineData: { - mimeType: match[1], - data: match[2], - }, - }) - } - } - } - } - - // Handle tool calls - if (message.tool_calls) { - for (const toolCall of message.tool_calls) { - parts.push({ - functionCall: { - name: toolCall.function.name, - args: - toolCall.function.arguments ? - (JSON.parse(toolCall.function.arguments) as Record< - string, - unknown - >) - : {}, - }, - }) - } - } - - return { - parts, - role: "model", - } -} - -function mapOpenAIFinishReasonToGemini( - finishReason: string | null, -): GeminiCandidate["finishReason"] { - switch (finishReason) { - case "stop": { - return "STOP" - } - case "length": { - return "MAX_TOKENS" - } - case "content_filter": { - return "SAFETY" - } - case "tool_calls": { - return "STOP" - } // Gemini doesn't have a specific tool_calls finish reason - default: { - return "FINISH_REASON_UNSPECIFIED" - } - } -} - -// Utility functions - -function generateToolCallId(functionName: string): string { - return `call_${functionName}_${Date.now()}_${Math.random().toString(36).slice(2, 11)}` -} - -// Helper function to process tool calls in streaming chunks -function processToolCalls( - toolCalls: Array<{ - index: number - id?: string - type?: "function" - function?: { - name?: string - arguments?: string - } - }>, -): Array { - const parts: Array = [] - - for (const toolCall of toolCalls) { - if (!toolCall.function?.name) { - continue - } - - let args: Record - try { - args = JSON.parse(toolCall.function.arguments || "{}") as Record< - string, - unknown - > - } catch { - // In streaming, arguments might be incomplete JSON - // Skip this chunk and wait for complete arguments - continue - } - - parts.push({ - functionCall: { - name: toolCall.function.name, - args, - }, - }) - } - - return parts -} - -// Helper function to create usage metadata -function createUsageMetadata(chunk: ChatCompletionChunk): GeminiUsageMetadata { - return { - promptTokenCount: chunk.usage?.prompt_tokens || 0, - candidatesTokenCount: chunk.usage?.completion_tokens || 0, - totalTokenCount: chunk.usage?.total_tokens || 0, - } -} - -// Stream translation: OpenAI Chunk -> Gemini Stream Response -export function translateOpenAIChunkToGemini(chunk: ChatCompletionChunk): { - candidates: Array - usageMetadata?: GeminiUsageMetadata -} | null { - if (chunk.choices.length === 0) { - return null - } - - const choice = chunk.choices[0] - const parts: Array = [] - - if (choice.delta.content) { - parts.push({ text: choice.delta.content }) - } - - if (choice.delta.tool_calls) { - parts.push(...processToolCalls(choice.delta.tool_calls)) - } - - if (parts.length === 0 && !choice.finish_reason) { - return null - } - - const candidate: GeminiCandidate = { - content: { - parts, - role: "model", - }, - finishReason: mapOpenAIFinishReasonToGemini(choice.finish_reason), - index: choice.index, - } - - const response: { - candidates: Array - usageMetadata?: GeminiUsageMetadata - } = { - candidates: [candidate], - } - - if (choice.finish_reason) { - response.usageMetadata = createUsageMetadata(chunk) - } - - return response -} - -// Token counting translation - -export function translateGeminiCountTokensToOpenAI( - request: GeminiCountTokensRequest, - model?: string, -): ChatCompletionsPayload { - return { - model: model || "claude-sonnet-4", - messages: translateGeminiContentsToOpenAI( - request.contents, - request.systemInstruction, - ), - max_tokens: 1, // Minimal for token counting - tools: translateGeminiToolsToOpenAI(request.tools), - } -} - -export function translateTokenCountToGemini( - totalTokens: number, -): GeminiCountTokensResponse { - return { - totalTokens, - } -} diff --git a/src/routes/messages/gemini-types.ts b/src/routes/messages/gemini-types.ts deleted file mode 100644 index 98ba5aff8..000000000 --- a/src/routes/messages/gemini-types.ts +++ /dev/null @@ -1,123 +0,0 @@ -// Gemini API Types - -export interface GeminiRequest { - contents: Array - tools?: Array - toolConfig?: GeminiToolConfig - safetySettings?: Array - systemInstruction?: GeminiContent - generationConfig?: GeminiGenerationConfig -} - -export interface GeminiContent { - parts: Array - role?: "user" | "model" -} - -export type GeminiPart = - | GeminiTextPart - | GeminiInlineDataPart - | GeminiFunctionCallPart - | GeminiFunctionResponsePart - -export interface GeminiTextPart { - text: string -} - -export interface GeminiInlineDataPart { - inlineData: { - mimeType: string - data: string - } -} - -export interface GeminiFunctionCallPart { - functionCall: { - name: string - args: Record - } -} - -export interface GeminiFunctionResponsePart { - functionResponse: { - name: string - response: Record - } -} - -export interface GeminiTool { - functionDeclarations: Array -} - -export interface GeminiFunctionDeclaration { - name: string - description?: string - parameters: Record -} - -export interface GeminiToolConfig { - functionCallingConfig: { - mode: "AUTO" | "ANY" | "NONE" - allowedFunctionNames?: Array - } -} - -export interface GeminiSafetySetting { - category: string - threshold: string -} - -export interface GeminiGenerationConfig { - stopSequences?: Array - temperature?: number - maxOutputTokens?: number - topP?: number - topK?: number -} - -// Response types -export interface GeminiResponse { - candidates: Array - usageMetadata?: GeminiUsageMetadata -} - -export interface GeminiCandidate { - content: GeminiContent - finishReason?: - | "FINISH_REASON_UNSPECIFIED" - | "STOP" - | "MAX_TOKENS" - | "SAFETY" - | "RECITATION" - | "OTHER" - index: number - safetyRatings?: Array -} - -export interface GeminiSafetyRating { - category: string - probability: string -} - -export interface GeminiUsageMetadata { - promptTokenCount: number - candidatesTokenCount: number - totalTokenCount: number -} - -// Token counting types -export interface GeminiCountTokensRequest { - contents: Array - tools?: Array - systemInstruction?: GeminiContent -} - -export interface GeminiCountTokensResponse { - totalTokens: number -} - -// Streaming types -export interface GeminiStreamResponse { - candidates?: Array - usageMetadata?: GeminiUsageMetadata -} From 8f9f44929fa79993345931b1fe7a2187566cac03 Mon Sep 17 00:00:00 2001 From: cpf Date: Sat, 27 Sep 2025 00:33:08 +0800 Subject: [PATCH 06/16] Add comprehensive tests for content generation and translation features - Introduced test types for mocking server responses and payloads. - Implemented translation coverage tests for OpenAI to Gemini response translation. - Added translation tests to validate tool configuration and content processing. - Created validation and routing tests to ensure proper error handling and request validation. - Enhanced existing tests to cover various edge cases and ensure robust functionality. --- src/lib/debug-logger.ts | 179 +++++++ src/routes/generate-content/handler.ts | 9 + src/routes/generate-content/route.ts | 21 +- src/routes/generate-content/translation.ts | 309 ++++++++++-- src/routes/generate-content/types.ts | 44 +- src/routes/generate-content/utils.ts | 57 +-- tests/@types/server-with-query.d.ts | 4 + tests/generate-content/_test-utils.ts | 110 ++++ .../core-functionality.test.ts | 220 ++++++++ tests/generate-content/route-routing.test.ts | 162 ++++++ .../stream-tool-call-accumulator.test.ts | 240 +++++++++ tests/generate-content/streaming.test.ts | 236 +++++++++ tests/generate-content/test-types.ts | 73 +++ .../translation-coverage.test.ts | 470 ++++++++++++++++++ .../translation-response-coverage.test.ts | 132 +++++ tests/generate-content/translation.test.ts | 320 ++++++++++++ .../validation-and-routing.test.ts | 236 +++++++++ 17 files changed, 2694 insertions(+), 128 deletions(-) create mode 100644 src/lib/debug-logger.ts create mode 100644 tests/@types/server-with-query.d.ts create mode 100644 tests/generate-content/_test-utils.ts create mode 100644 tests/generate-content/core-functionality.test.ts create mode 100644 tests/generate-content/route-routing.test.ts create mode 100644 tests/generate-content/stream-tool-call-accumulator.test.ts create mode 100644 tests/generate-content/streaming.test.ts create mode 100644 tests/generate-content/test-types.ts create mode 100644 tests/generate-content/translation-coverage.test.ts create mode 100644 tests/generate-content/translation-response-coverage.test.ts create mode 100644 tests/generate-content/translation.test.ts create mode 100644 tests/generate-content/validation-and-routing.test.ts diff --git a/src/lib/debug-logger.ts b/src/lib/debug-logger.ts new file mode 100644 index 000000000..e51c723ae --- /dev/null +++ b/src/lib/debug-logger.ts @@ -0,0 +1,179 @@ +import { existsSync, mkdirSync } from "node:fs" +import { writeFile } from "node:fs/promises" +import { join } from "node:path" + +import type { GeminiRequest } from "~/routes/generate-content/types" +import type { + ChatCompletionsPayload, + ChatCompletionResponse, +} from "~/services/copilot/create-chat-completions" + +interface DebugLogData { + timestamp: string + requestId: string + originalGeminiPayload: GeminiRequest + translatedOpenAIPayload: ChatCompletionsPayload | null + error?: string + processingTime?: number +} + +export class DebugLogger { + private static instance: DebugLogger | undefined + private logDir: string + + private constructor() { + this.logDir = process.env.DEBUG_LOG_DIR || join(process.cwd(), "debug-logs") + this.ensureLogDir() + } + + static getInstance(): DebugLogger { + if (!DebugLogger.instance) { + DebugLogger.instance = new DebugLogger() + } + return DebugLogger.instance + } + + private ensureLogDir(): void { + if (!existsSync(this.logDir)) { + mkdirSync(this.logDir, { recursive: true }) + } + } + + private generateLogFileName(requestId: string): string { + const timestamp = new Date().toISOString().replaceAll(/[:.]/g, "-") + return join(this.logDir, `debug-gemini-${timestamp}-${requestId}.log`) + } + + async logRequest(data: { + requestId: string + geminiPayload: GeminiRequest + openAIPayload?: ChatCompletionsPayload | null + error?: string + processingTime?: number + }): Promise { + const logData: DebugLogData = { + timestamp: new Date().toISOString(), + requestId: data.requestId, + originalGeminiPayload: data.geminiPayload, + translatedOpenAIPayload: data.openAIPayload ?? null, + error: data.error, + processingTime: data.processingTime, + } + + const logPath = this.generateLogFileName(data.requestId) + + try { + await writeFile(logPath, JSON.stringify(logData, null, 2), "utf8") + console.log(`[DEBUG] Logged request data to: ${logPath}`) + } catch (writeError) { + console.error(`[DEBUG] Failed to write log file ${logPath}:`, writeError) + } + } + + // For backward compatibility during development + static async logGeminiRequest( + geminiPayload: GeminiRequest, + openAIPayload?: ChatCompletionsPayload, + error?: string, + ): Promise { + const logger = DebugLogger.getInstance() + const requestId = Math.random().toString(36).slice(2, 8) + await logger.logRequest({ requestId, geminiPayload, openAIPayload, error }) + } + + // Log GitHub Copilot API Response + static async logCopilotResponse( + response: ChatCompletionResponse, + context?: string, + ): Promise { + const logger = DebugLogger.getInstance() + const requestId = Math.random().toString(36).slice(2, 8) + const timestamp = new Date().toISOString().replaceAll(/[:.]/g, "-") + const logPath = join( + logger.logDir, + `debug-copilot-response-${timestamp}-${requestId}.log`, + ) + + const logData = { + timestamp: new Date().toISOString(), + context: context || "GitHub Copilot API Response", + response, + } + + try { + await writeFile(logPath, JSON.stringify(logData, null, 2), "utf8") + console.log(`[DEBUG] Logged Copilot response to: ${logPath}`) + } catch (writeError) { + console.error( + `[DEBUG] Failed to write Copilot response log file ${logPath}:`, + writeError, + ) + } + } + + // Log any object for debugging purposes + static async logDebugData( + data: unknown, + context: string, + filePrefix = "debug-data", + ): Promise { + const logger = DebugLogger.getInstance() + const requestId = Math.random().toString(36).slice(2, 8) + const timestamp = new Date().toISOString().replaceAll(/[:.]/g, "-") + const logPath = join( + logger.logDir, + `${filePrefix}-${timestamp}-${requestId}.log`, + ) + + const logData = { + timestamp: new Date().toISOString(), + context, + data, + } + + try { + await writeFile(logPath, JSON.stringify(logData, null, 2), "utf8") + console.log(`[DEBUG] Logged ${context} to: ${logPath}`) + } catch (writeError) { + console.error( + `[DEBUG] Failed to write debug log file ${logPath}:`, + writeError, + ) + } + } + + // Log original and translated response comparison + static async logResponseComparison( + originalResponse: unknown, + translatedResponse: unknown, + options: { context: string; filePrefix?: string } = { + context: "Response Comparison", + }, + ): Promise { + const { context, filePrefix = "debug-comparison" } = options + const logger = DebugLogger.getInstance() + const requestId = Math.random().toString(36).slice(2, 8) + const timestamp = new Date().toISOString().replaceAll(/[:.]/g, "-") + const logPath = join( + logger.logDir, + `${filePrefix}-${timestamp}-${requestId}.log`, + ) + + const logData = { + timestamp: new Date().toISOString(), + context, + originalResponse, + translatedResponse, + } + + try { + await writeFile(logPath, JSON.stringify(logData, null, 2), "utf8") + console.log(`[DEBUG] Logged ${context} comparison to: ${logPath}`) + } catch (writeError) { + console.error( + `[DEBUG] Failed to write comparison log file ${logPath}:`, + writeError, + ) + } + } +} diff --git a/src/routes/generate-content/handler.ts b/src/routes/generate-content/handler.ts index 0f5d93dc8..af0f2d3fd 100644 --- a/src/routes/generate-content/handler.ts +++ b/src/routes/generate-content/handler.ts @@ -4,6 +4,7 @@ import type { SSEStreamingApi } from "hono/streaming" import { streamSSE } from "hono/streaming" import { awaitApproval } from "~/lib/approval" +import { DebugLogger } from "~/lib/debug-logger" import { checkRateLimit } from "~/lib/rate-limit" import { state } from "~/lib/state" import { getTokenCount } from "~/lib/tokenizer" @@ -295,6 +296,14 @@ export async function handleGeminiStreamGeneration(c: Context) { const openAIPayload = translateGeminiToOpenAIStream(geminiPayload, model) + // Log request for debugging (async, non-blocking) - only if debug logging is enabled + if (process.env.DEBUG_GEMINI_REQUESTS === "true") { + DebugLogger.logGeminiRequest(geminiPayload, openAIPayload).catch( + (error: unknown) => { + console.error("[DEBUG] Failed to log request:", error) + }, + ) + } if (state.manualApprove) { await awaitApproval() } diff --git a/src/routes/generate-content/route.ts b/src/routes/generate-content/route.ts index ca57bda56..64cb48a45 100644 --- a/src/routes/generate-content/route.ts +++ b/src/routes/generate-content/route.ts @@ -8,13 +8,25 @@ import { handleGeminiCountTokens, } from "./handler" +function isStreamGenerate(url: string): boolean { + return url.includes(":streamGenerateContent") +} +function isCountTokens(url: string): boolean { + return url.includes(":countTokens") +} +function isGenerate(url: string): boolean { + return ( + url.includes(":generateContent") && !url.includes(":streamGenerateContent") + ) +} + const router = new Hono() // Streaming generation endpoint // POST /v1beta/models/{model}:streamGenerateContent router.post("/v1beta/models/*", async (c, next) => { const url = c.req.url - if (url.includes(":streamGenerateContent")) { + if (isStreamGenerate(url)) { try { return await handleGeminiStreamGeneration(c) } catch (error) { @@ -28,7 +40,7 @@ router.post("/v1beta/models/*", async (c, next) => { // POST /v1beta/models/{model}:countTokens router.post("/v1beta/models/*", async (c, next) => { const url = c.req.url - if (url.includes(":countTokens")) { + if (isCountTokens(url)) { try { return await handleGeminiCountTokens(c) } catch (error) { @@ -42,10 +54,7 @@ router.post("/v1beta/models/*", async (c, next) => { // POST /v1beta/models/{model}:generateContent router.post("/v1beta/models/*", async (c, next) => { const url = c.req.url - if ( - url.includes(":generateContent") - && !url.includes(":streamGenerateContent") - ) { + if (isGenerate(url)) { try { return await handleGeminiGeneration(c) } catch (error) { diff --git a/src/routes/generate-content/translation.ts b/src/routes/generate-content/translation.ts index 7adc9a3cb..c3ea83a81 100644 --- a/src/routes/generate-content/translation.ts +++ b/src/routes/generate-content/translation.ts @@ -1,3 +1,4 @@ +import { DebugLogger } from "~/lib/debug-logger" import { type ChatCompletionResponse, type ChatCompletionChunk, @@ -29,31 +30,45 @@ function mapGeminiModelToCopilot(geminiModel: string): string { const modelMap: Record = { "gemini-2.5-flash": "gemini-2.0-flash-001", // Map to supported Gemini model "gemini-2.0-flash": "gemini-2.0-flash-001", // Map to full model name + "gemini-2.5-flash-lite": "gemini-2.0-flash-001", // Map to full model name } return modelMap[geminiModel] || geminiModel // Return original if supported } +function selectTools( + geminiTools?: Array, + contents?: Array< + | GeminiContent + | Array<{ + functionResponse: { id?: string; name: string; response: unknown } + }> + >, +): Array | undefined { + return ( + translateGeminiToolsToOpenAI(geminiTools) + || (contents ? synthesizeToolsFromContents(contents) : undefined) + ) +} + // Request translation: Gemini -> OpenAI export function translateGeminiToOpenAINonStream( payload: GeminiRequest, model: string, ): ChatCompletionsPayload { - const tools = - translateGeminiToolsToOpenAI(payload.tools) - || synthesizeToolsFromContents(payload.contents) + const tools = selectTools(payload.tools, payload.contents) const result = { model: mapGeminiModelToCopilot(model), messages: translateGeminiContentsToOpenAI( payload.contents, payload.systemInstruction, ), - max_tokens: payload.generationConfig?.maxOutputTokens || 4096, - stop: payload.generationConfig?.stopSequences, + max_tokens: (payload.generationConfig?.maxOutputTokens as number) || 4096, + stop: payload.generationConfig?.stopSequences as Array | undefined, stream: false, - temperature: payload.generationConfig?.temperature, - top_p: payload.generationConfig?.topP, + temperature: payload.generationConfig?.temperature as number | undefined, + top_p: payload.generationConfig?.topP as number | undefined, tools, tool_choice: tools ? translateGeminiToolConfigToOpenAI(payload.toolConfig) : undefined, @@ -66,20 +81,18 @@ export function translateGeminiToOpenAIStream( payload: GeminiRequest, model: string, ): ChatCompletionsPayload { - const tools = - translateGeminiToolsToOpenAI(payload.tools) - || synthesizeToolsFromContents(payload.contents) + const tools = selectTools(payload.tools, payload.contents) const result = { model: mapGeminiModelToCopilot(model), messages: translateGeminiContentsToOpenAI( payload.contents, payload.systemInstruction, ), - max_tokens: payload.generationConfig?.maxOutputTokens || 4096, - stop: payload.generationConfig?.stopSequences, + max_tokens: (payload.generationConfig?.maxOutputTokens as number) || 4096, + stop: payload.generationConfig?.stopSequences as Array | undefined, stream: true, - temperature: payload.generationConfig?.temperature, - top_p: payload.generationConfig?.topP, + temperature: payload.generationConfig?.temperature as number | undefined, + top_p: payload.generationConfig?.topP as number | undefined, tools, tool_choice: tools ? translateGeminiToolConfigToOpenAI(payload.toolConfig) : undefined, @@ -99,14 +112,24 @@ function processFunctionResponseArray( for (const responseItem of responseArray) { if ("functionResponse" in responseItem) { const functionName = responseItem.functionResponse.name - const toolCallId = pendingToolCalls.get(functionName) - if (toolCallId) { + // Find tool call ID by searching through the map + let matchedToolCallId: string | undefined + for (const [ + toolCallId, + mappedFunctionName, + ] of pendingToolCalls.entries()) { + if (mappedFunctionName === functionName) { + matchedToolCallId = toolCallId + break + } + } + if (matchedToolCallId) { messages.push({ role: "tool", - tool_call_id: toolCallId, + tool_call_id: matchedToolCallId, content: JSON.stringify(responseItem.functionResponse.response), }) - pendingToolCalls.delete(functionName) + pendingToolCalls.delete(matchedToolCallId) } } } @@ -138,14 +161,21 @@ function processFunctionResponses( ): void { for (const funcResponse of functionResponses) { const functionName = funcResponse.functionResponse.name - const toolCallId = pendingToolCalls.get(functionName) - if (toolCallId) { + // Find tool call ID by searching through the map + let matchedToolCallId: string | undefined + for (const [toolCallId, mappedFunctionName] of pendingToolCalls.entries()) { + if (mappedFunctionName === functionName) { + matchedToolCallId = toolCallId + break + } + } + if (matchedToolCallId) { messages.push({ role: "tool", - tool_call_id: toolCallId, + tool_call_id: matchedToolCallId, content: JSON.stringify(funcResponse.functionResponse.response), }) - pendingToolCalls.delete(functionName) + pendingToolCalls.delete(matchedToolCallId) } } } @@ -163,7 +193,8 @@ function processFunctionCalls(options: { const toolCalls = functionCalls.map((call) => { const toolCallId = generateToolCallId(call.functionCall.name) // Remember this tool call for later matching with responses - pendingToolCalls.set(call.functionCall.name, toolCallId) + // Use tool_call_id as key to avoid duplicate function name overwrites + pendingToolCalls.set(toolCallId, call.functionCall.name) return { id: toolCallId, @@ -195,6 +226,8 @@ function mergeConsecutiveSameRoleMessages( && lastMessage.role === message.role && !lastMessage.tool_calls && !message.tool_calls + && !(lastMessage as { tool_call_id?: string }).tool_call_id // Don't merge tool responses + && !(message as { tool_call_id?: string }).tool_call_id // Don't merge tool responses ) { // Merge with previous message of same role if ( @@ -245,7 +278,7 @@ function translateGeminiContentsToOpenAI( systemInstruction?: GeminiContent, ): Array { const messages: Array = [] - const pendingToolCalls = new Map() // function name -> tool_call_id + const pendingToolCalls = new Map() // tool_call_id -> function_name // Add system instruction first if present if (systemInstruction) { @@ -297,8 +330,11 @@ function translateGeminiContentsToOpenAI( // Post-process: Remove incomplete assistant messages from cancelled tool calls removeIncompleteAssistantMessages(messages) + // Post-process: Deduplicate tool responses (remove duplicate tool_call_ids) + const matchedMessages = ensureToolCallResponseMatch(messages) + // Post-process: Merge consecutive messages with same role (based on LiteLLM research) - return mergeConsecutiveSameRoleMessages(messages) + return mergeConsecutiveSameRoleMessages(matchedMessages) } function synthesizeToolsFromContents( @@ -343,10 +379,14 @@ function translateGeminiContentToOpenAI( if ("text" in part) { contentParts.push({ type: "text", text: part.text }) } else if ("inlineData" in part) { + // Handle inline data for images - this is a legacy format + const partWithInlineData = part as { + inlineData: { mimeType: string; data: string } + } contentParts.push({ type: "image_url", image_url: { - url: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`, + url: `data:${partWithInlineData.inlineData.mimeType};base64,${partWithInlineData.inlineData.data}`, }, }) } @@ -454,25 +494,62 @@ function translateGeminiToolConfigToOpenAI( // Response translation: OpenAI -> Gemini +// Helper function to deduplicate tool responses - remove duplicate tool_call_ids +// The problem was our logic was CREATING duplicates instead of preventing them + +function ensureToolCallResponseMatch(messages: Array): Array { + const result: Array = [] + const seenToolCallIds = new Set() // Track processed tool_call_ids to avoid duplicates + + for (const message of messages) { + if (message.role === "tool" && message.tool_call_id) { + const toolCallId = message.tool_call_id + + // Only keep the FIRST response for each tool_call_id (deduplicate) + if (!seenToolCallIds.has(toolCallId)) { + seenToolCallIds.add(toolCallId) + result.push(message) + } + // Skip any duplicate responses for the same tool_call_id + } else { + // Keep all non-tool messages as-is + result.push(message) + } + } + + return result +} + export function translateOpenAIToGemini( response: ChatCompletionResponse, ): GeminiResponse { - const candidates: Array = response.choices.map( - (choice, index) => ({ + const result = { + candidates: response.choices.map((choice, index) => ({ content: translateOpenAIMessageToGeminiContent(choice.message), finishReason: mapOpenAIFinishReasonToGemini(choice.finish_reason), index, - }), - ) - - return { - candidates, + })), usageMetadata: { promptTokenCount: response.usage?.prompt_tokens || 0, candidatesTokenCount: response.usage?.completion_tokens || 0, totalTokenCount: response.usage?.total_tokens || 0, }, } + + // Debug: Log original GitHub Copilot response and translated Gemini response for comparison + if (process.env.DEBUG_GEMINI_REQUESTS === "true") { + DebugLogger.logResponseComparison(response, result, { + context: "Non-Stream Response Translation", + filePrefix: "debug-nonstream-comparison", + }).catch((error: unknown) => { + console.error( + "[DEBUG] Failed to log non-stream response comparison:", + error, + ) + }) + } + + return result } function translateOpenAIMessageToGeminiContent( @@ -507,6 +584,13 @@ function translateOpenAIMessageToGeminiContent( // Handle tool calls if (message.tool_calls) { for (const toolCall of message.tool_calls) { + // Debug: Log tool call arguments to verify what GitHub Copilot returns + if (process.env.DEBUG_GEMINI_REQUESTS === "true") { + console.log( + `[DEBUG] Tool call - name: ${toolCall.function.name}, arguments: "${toolCall.function.arguments}", type: ${typeof toolCall.function.arguments}, truthy: ${Boolean(toolCall.function.arguments)}`, + ) + } + parts.push({ functionCall: { name: toolCall.function.name, @@ -534,7 +618,94 @@ function generateToolCallId(functionName: string): string { return `call_${functionName}_${Date.now()}_${Math.random().toString(36).slice(2, 11)}` } -// Helper function to process tool calls in streaming chunks +// Global accumulator for streaming tool call arguments +const streamingToolCallAccumulator = new Map< + number, + { + name: string + arguments: string + id?: string + } +>() + +// Helper function to try parsing and creating a function call +function tryCreateFunctionCall( + name: string, + argumentsStr: string, +): GeminiPart | null { + try { + const args = JSON.parse(argumentsStr) as Record + return { + functionCall: { + name, + args, + }, + } + } catch { + return null + } +} + +// Helper function to handle tool call with function name +function handleToolCallWithName(toolCall: { + index: number + id?: string + function: { + name: string + arguments?: string + } +}): GeminiPart | null { + const accumulatedArgs = toolCall.function.arguments || "" + + streamingToolCallAccumulator.set(toolCall.index, { + name: toolCall.function.name, + arguments: accumulatedArgs, + id: toolCall.id, + }) + + // If we already have arguments, try to process immediately (for non-streaming models like Gemini) + if (accumulatedArgs) { + const functionCall = tryCreateFunctionCall( + toolCall.function.name, + accumulatedArgs, + ) + if (functionCall) { + // Clear the accumulator for this index since we've successfully processed it + streamingToolCallAccumulator.delete(toolCall.index) + return functionCall + } + } + + return null +} + +// Helper function to handle tool call argument accumulation +function handleToolCallAccumulation(toolCall: { + index: number + function?: { + arguments?: string + } +}): GeminiPart | null { + const existingAccumulated = streamingToolCallAccumulator.get(toolCall.index) + + if (existingAccumulated && toolCall.function?.arguments) { + existingAccumulated.arguments += toolCall.function.arguments + + const functionCall = tryCreateFunctionCall( + existingAccumulated.name, + existingAccumulated.arguments, + ) + if (functionCall) { + // Clear the accumulator for this index since we've successfully processed it + streamingToolCallAccumulator.delete(toolCall.index) + return functionCall + } + } + + return null +} + +// Helper function to process tool calls in streaming chunks with argument accumulation function processToolCalls( toolCalls: Array<{ index: number @@ -549,28 +720,34 @@ function processToolCalls( const parts: Array = [] for (const toolCall of toolCalls) { - if (!toolCall.function?.name) { - continue + // Debug: Log streaming tool call arguments to verify what GitHub Copilot returns + if (process.env.DEBUG_GEMINI_REQUESTS === "true") { + console.log( + `[DEBUG STREAM] Tool call - name: ${toolCall.function?.name}, arguments: "${toolCall.function?.arguments}", type: ${typeof toolCall.function?.arguments}, truthy: ${Boolean(toolCall.function?.arguments)}`, + ) } - let args: Record - try { - args = JSON.parse(toolCall.function.arguments || "{}") as Record< - string, - unknown - > - } catch { - // In streaming, arguments might be incomplete JSON - // Skip this chunk and wait for complete arguments + // If this chunk has a function name, it's the start of a new tool call + if (toolCall.function?.name && toolCall.function.name.trim() !== "") { + const functionCall = handleToolCallWithName({ + index: toolCall.index, + id: toolCall.id, + function: { + name: toolCall.function.name, + arguments: toolCall.function.arguments, + }, + }) + if (functionCall) { + parts.push(functionCall) + } continue } - parts.push({ - functionCall: { - name: toolCall.function.name, - args, - }, - }) + // If we have existing accumulated data and this chunk has arguments, append them + const functionCall = handleToolCallAccumulation(toolCall) + if (functionCall) { + parts.push(functionCall) + } } return parts @@ -711,6 +888,22 @@ export function translateOpenAIChunkToGemini(chunk: ChatCompletionChunk): { return null } + // Additional validation - if we only have function call parts with empty names, + // skip this chunk entirely to prevent invalid tool call responses + const hasOnlyEmptyToolCalls = + parts.length > 0 + && parts.every((part) => { + if ("functionCall" in part) { + return !part.functionCall.name || part.functionCall.name.trim() === "" + } + return false + }) + && parts.some((part) => "functionCall" in part) + + if (hasOnlyEmptyToolCalls && !choice.finish_reason) { + return null + } + const shouldInclude = shouldIncludeFinishReason(choice) const mappedFinishReason = shouldInclude ? @@ -724,6 +917,16 @@ export function translateOpenAIChunkToGemini(chunk: ChatCompletionChunk): { ) const response = buildGeminiResponse(candidate, shouldInclude, chunk) + // Debug: Log original GitHub Copilot chunk and translated Gemini chunk for comparison + if (process.env.DEBUG_GEMINI_REQUESTS === "true") { + DebugLogger.logResponseComparison(chunk, response, { + context: "Streaming Chunk Translation", + filePrefix: "debug-stream-comparison", + }).catch((error: unknown) => { + console.error("[DEBUG] Failed to log streaming chunk comparison:", error) + }) + } + return response } @@ -733,9 +936,7 @@ export function translateGeminiCountTokensToOpenAI( request: GeminiCountTokensRequest, model: string, ): ChatCompletionsPayload { - const tools = - translateGeminiToolsToOpenAI(request.tools) - || synthesizeToolsFromContents(request.contents) + const tools = selectTools(request.tools, request.contents) return { model: mapGeminiModelToCopilot(model), messages: translateGeminiContentsToOpenAI( diff --git a/src/routes/generate-content/types.ts b/src/routes/generate-content/types.ts index 8de67138d..8e893afe1 100644 --- a/src/routes/generate-content/types.ts +++ b/src/routes/generate-content/types.ts @@ -4,9 +4,9 @@ export interface GeminiRequest { contents: Array tools?: Array toolConfig?: GeminiToolConfig - safetySettings?: Array + safetySettings?: Array> systemInstruction?: GeminiContent - generationConfig?: GeminiGenerationConfig + generationConfig?: Record } export interface GeminiContent { @@ -16,15 +16,15 @@ export interface GeminiContent { export type GeminiPart = | GeminiTextPart - | GeminiInlineDataPart | GeminiFunctionCallPart | GeminiFunctionResponsePart + | GeminiInlineDataPart export interface GeminiTextPart { text: string } -export interface GeminiInlineDataPart { +interface GeminiInlineDataPart { inlineData: { mimeType: string data: string @@ -51,48 +51,25 @@ export interface GeminiTool { urlContext?: Record } -export interface GeminiFunctionDeclaration { +interface GeminiFunctionDeclaration { name: string description?: string parameters?: Record parametersJsonSchema?: Record } -export interface GeminiToolConfig { +interface GeminiToolConfig { functionCallingConfig: { mode: "AUTO" | "ANY" | "NONE" allowedFunctionNames?: Array } } -export interface GeminiSafetySetting { - category: string - threshold: string -} - -export interface GeminiGenerationConfig { - stopSequences?: Array - temperature?: number - maxOutputTokens?: number - topP?: number - topK?: number -} - // Response types export interface GeminiResponse { candidates: Array usageMetadata?: GeminiUsageMetadata - promptFeedback?: GeminiPromptFeedback -} - -export interface GeminiPromptFeedback { - blockReason?: - | "BLOCK_REASON_UNSPECIFIED" - | "SAFETY" - | "OTHER" - | "BLOCKLIST" - | "PROHIBITED_CONTENT" - safetyRatings?: Array + promptFeedback?: Record } export interface GeminiCandidate { @@ -113,12 +90,7 @@ export interface GeminiCandidate { | "UNEXPECTED_TOOL_CALL" | "TOO_MANY_TOOL_CALLS" index: number - safetyRatings?: Array -} - -export interface GeminiSafetyRating { - category: string - probability: string + safetyRatings?: Array> } export interface GeminiUsageMetadata { diff --git a/src/routes/generate-content/utils.ts b/src/routes/generate-content/utils.ts index 88f1c89b5..1f3fa7dc0 100644 --- a/src/routes/generate-content/utils.ts +++ b/src/routes/generate-content/utils.ts @@ -1,50 +1,43 @@ import { type GeminiCandidate } from "./types" +const OpenAIFinish = { + stop: "stop", + length: "length", + content_filter: "content_filter", + tool_calls: "tool_calls", +} as const + +const GeminiFinish = { + FINISH_REASON_UNSPECIFIED: "FINISH_REASON_UNSPECIFIED", + STOP: "STOP", + MAX_TOKENS: "MAX_TOKENS", + SAFETY: "SAFETY", + RECITATION: "RECITATION", + BLOCKLIST: "BLOCKLIST", + PROHIBITED_CONTENT: "PROHIBITED_CONTENT", + SPII: "SPII", + IMAGE_SAFETY: "IMAGE_SAFETY", + MALFORMED_FUNCTION_CALL: "MALFORMED_FUNCTION_CALL", +} as const + export function mapOpenAIFinishReasonToGemini( finishReason: string | null, ): GeminiCandidate["finishReason"] { switch (finishReason) { - case "stop": { + case OpenAIFinish.stop: { return "STOP" } - case "length": { + case OpenAIFinish.length: { return "MAX_TOKENS" } - case "content_filter": { + case OpenAIFinish.content_filter: { return "SAFETY" } - case "tool_calls": { + case OpenAIFinish.tool_calls: { return "STOP" // Gemini doesn't have a specific tool_calls finish reason, map to STOP } default: { - return "FINISH_REASON_UNSPECIFIED" - } - } -} - -// Add the reverse mapping - Gemini → OpenAI (based on LiteLLM research) -export function mapGeminiFinishReasonToOpenAI( - finishReason: string | undefined, -): "stop" | "length" | "content_filter" | "tool_calls" { - switch (finishReason) { - case "STOP": - case "FINISH_REASON_UNSPECIFIED": - case "MALFORMED_FUNCTION_CALL": { - return "stop" - } - case "MAX_TOKENS": { - return "length" - } - case "SAFETY": - case "RECITATION": - case "BLOCKLIST": - case "PROHIBITED_CONTENT": - case "SPII": - case "IMAGE_SAFETY": { - return "content_filter" - } - default: { - return "stop" + return GeminiFinish.FINISH_REASON_UNSPECIFIED } } } diff --git a/tests/@types/server-with-query.d.ts b/tests/@types/server-with-query.d.ts new file mode 100644 index 000000000..a8f40adf6 --- /dev/null +++ b/tests/@types/server-with-query.d.ts @@ -0,0 +1,4 @@ +// Allow importing "~/server?foo" variants in tests without impacting runtime behavior. +declare module "~/server?*" { + export const server: import("hono").Hono +} diff --git a/tests/generate-content/_test-utils.ts b/tests/generate-content/_test-utils.ts new file mode 100644 index 000000000..11a1dcdf1 --- /dev/null +++ b/tests/generate-content/_test-utils.ts @@ -0,0 +1,110 @@ +import { mock } from "bun:test" + +import type { + TestServer, + MockChatCompletionsModule, + MockRateLimitModule, + MockTokenCountModule, +} from "./test-types" + +export function asyncIterableFrom( + events: Array<{ data?: string }>, +): AsyncIterable<{ data: string }> { + return { + [Symbol.asyncIterator]() { + let i = 0 + return { + next(): Promise> { + if (i < events.length) { + const event = events[i++] + return Promise.resolve({ + value: { data: event.data ?? "" }, + done: false, + }) + } + return Promise.resolve({ + value: undefined as unknown as { data: string }, + done: true, + }) + }, + } + }, + } +} + +export function createMockChatCompletions(events: Array<{ data?: string }>) { + return mock.module( + "~/services/copilot/create-chat-completions", + (): MockChatCompletionsModule => ({ + createChatCompletions: () => asyncIterableFrom(events), + }), + ) +} + +export function createMockRateLimit() { + return mock.module( + "~/lib/rate-limit", + (): MockRateLimitModule => ({ + checkRateLimit: (_: unknown) => {}, + }), + ) +} + +export function createMockTokenCount(tokens: { + input: number + output: number +}) { + return mock.module( + "~/services/copilot/get-token-count", + (): MockTokenCountModule => ({ + getTokenCount: () => tokens, + }), + ) +} + +export async function makeStreamRequest( + path: string, + body: Record, + queryString?: string, +): Promise { + const serverModule = (await import(`~/server?${queryString}`)) as { + server: TestServer + } + return serverModule.server.request(path, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }) +} + +export async function makeRequest( + path: string, + body: Record, + queryString?: string, +): Promise { + const serverModule = (await import(`~/server?${queryString}`)) as { + server: TestServer + } + return serverModule.server.request(path, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }) +} + +export const commonResponseData = { + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, +} + +export const sampleGeminiRequest = { + contents: [{ role: "user", parts: [{ text: "Hello" }] }], +} + +export const sampleToolCall = { + index: 0, + type: "function", + function: { + name: "ReadFile", + arguments: '{"absolute_path": "/path/to/file.txt"}', + }, +} diff --git a/tests/generate-content/core-functionality.test.ts b/tests/generate-content/core-functionality.test.ts new file mode 100644 index 000000000..7f05608e8 --- /dev/null +++ b/tests/generate-content/core-functionality.test.ts @@ -0,0 +1,220 @@ +import { afterEach, expect, test, mock } from "bun:test" + +import type { TestServer } from "./test-types" + +import { createMockChatCompletions } from "./_test-utils" + +afterEach(() => { + mock.restore() +}) + +test("translates request and uses local tokenizer without downstream call", async () => { + let downstreamCalled = false + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => { + downstreamCalled = true + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + await mock.module("~/lib/tokenizer", () => ({ + getTokenCount: (_: unknown) => ({ input: 2, output: 3 }), + })) + + const { server } = (await import("~/server")) as { server: TestServer } + const res = await server.request("/v1beta/models/gemini-pro:countTokens", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }) + + expect(res.status).toBe(200) + const json = (await res.json()) as { totalTokens: number } + expect(json).toEqual({ totalTokens: 5 }) + expect(downstreamCalled).toBe(false) +}) + +test("maps finish_reason stop/length/content_filter/tool_calls correctly (non-stream)", async () => { + const finishCases = [ + { fr: "stop", expected: "STOP" }, + { fr: "length", expected: "MAX_TOKENS" }, + { fr: "content_filter", expected: "SAFETY" }, + { fr: "tool_calls", expected: "STOP" }, + ] + + let idx = 0 + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => { + const fr = finishCases[idx++].fr as + | "stop" + | "length" + | "content_filter" + | "tool_calls" + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: fr, + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 2, total_tokens: 3 }, + } + }, + })) + + const { server } = (await import("~/server")) as { server: TestServer } + for (const finishCase of finishCases) { + const res = await server.request( + "/v1beta/models/gemini-pro:generateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + expect(res.status).toBe(200) + const json = (await res.json()) as { + candidates: [{ finishReason: string }] + } + const got = json.candidates[0].finishReason + expect(got).toBe(finishCase.expected) + } +}) + +test("optional manual approval gate triggers before downstream call", async () => { + const calls: Array = [] + await mock.module("~/lib/state", () => ({ + state: { manualApprove: true }, + })) + await mock.module("~/lib/approval", () => ({ + awaitApproval: () => { + calls.push("approve") + }, + })) + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => { + calls.push("create") + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + const { server } = (await import("~/server")) as { server: TestServer } + const res = await server.request( + "/v1beta/models/gemini-pro:generateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + + expect(res.status).toBe(200) + expect(calls).toEqual(["approve", "create"]) +}) + +test("enforces rate limit before processing (non-stream)", async () => { + await mock.module("~/lib/rate-limit", () => ({ + checkRateLimit: () => { + throw new Error("Rate limited") + }, + })) + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => { + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + const { server } = (await import("~/server")) as { server: TestServer } + const res = await server.request( + "/v1beta/models/gemini-pro:generateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + + expect(res.status).toBe(500) + const json = (await res.json()) as { + error: { message: string; type: string } + } + expect(json).toEqual({ error: { message: "Rate limited", type: "error" } }) +}) + +test("enforces rate limit before stream", async () => { + await mock.module("~/lib/rate-limit", () => ({ + checkRateLimit: () => { + throw new Error("Rate limited stream") + }, + })) + await createMockChatCompletions([ + { + data: JSON.stringify({ + id: "c1", + choices: [{ index: 0, delta: { content: "x" }, finish_reason: null }], + }), + }, + { + data: JSON.stringify({ + id: "c1", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + }), + }, + { data: "[DONE]" }, + ]) + + const { server } = (await import("~/server")) as { server: TestServer } + const res = await server.request( + "/v1beta/models/gemini-pro:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + + expect(res.status).toBe(500) + const txt = await res.text() + expect(txt.includes("Rate limited stream")).toBe(true) +}) diff --git a/tests/generate-content/route-routing.test.ts b/tests/generate-content/route-routing.test.ts new file mode 100644 index 000000000..cb73173d8 --- /dev/null +++ b/tests/generate-content/route-routing.test.ts @@ -0,0 +1,162 @@ +import { afterEach, expect, test, mock } from "bun:test" + +function asyncIterableFrom(events: Array<{ data?: string }>) { + return { + [Symbol.asyncIterator]() { + let i = 0 + return { + next() { + if (i < events.length) + return Promise.resolve({ value: events[i++], done: false }) + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } +} + +afterEach(() => { + mock.restore() +}) + +test("routes to stream endpoint based on URL keyword", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (_: unknown) => + asyncIterableFrom([ + { + data: JSON.stringify({ + id: "c1", + choices: [ + { index: 0, delta: { content: "hi" }, finish_reason: null }, + ], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { + data: JSON.stringify({ + id: "c1", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { data: "[DONE]" }, + ]), + })) + await mock.module("~/lib/rate-limit", () => ({ + checkRateLimit: () => {}, + })) + const { server } = await import("~/server?route-routing") + const res = await server.request( + "/v1beta/models/gemini-pro:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + expect(res.status).toBe(200) + const ct = res.headers.get("content-type") || "" + expect(ct.includes("text/event-stream")).toBe(true) + const body = await res.text() + expect(body.includes("data:")).toBe(true) + expect(body.includes('"role":"model"')).toBe(true) +}) + +test("routes to countTokens endpoint based on URL keyword", async () => { + await mock.module("~/lib/tokenizer", () => ({ + getTokenCount: (_: unknown) => ({ input: 2, output: 3 }), + })) + await mock.module("~/lib/rate-limit", () => ({ + checkRateLimit: () => {}, + })) + const { server } = await import("~/server?route-routing") + const res = await server.request("/v1beta/models/gemini-pro:countTokens", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }) + expect(res.status).toBe(200) + const json = + (await res.json()) as import("~/routes/generate-content/types").GeminiCountTokensResponse + expect(json).toEqual({ totalTokens: 5 }) +}) + +test("routes to non-stream endpoint with path exclusivity", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (_: unknown) => ({ + id: "res-2", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }), + })) + await mock.module("~/lib/rate-limit", () => ({ + checkRateLimit: () => {}, + })) + const { server } = await import("~/server?route-routing") + const res = await server.request( + "/v1beta/models/gemini-pro:generateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + expect(res.status).toBe(200) + const ct = res.headers.get("content-type") || "" + expect(ct.includes("application/json")).toBe(true) + const json = + (await res.json()) as import("~/routes/generate-content/types").GeminiResponse + expect(Array.isArray(json.candidates)).toBe(true) +}) + +test("does NOT mis-route to non-stream endpoint", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (_: unknown) => + asyncIterableFrom([ + { + data: JSON.stringify({ + id: "c1", + choices: [ + { index: 0, delta: { content: "x" }, finish_reason: null }, + ], + }), + }, + { + data: JSON.stringify({ + id: "c1", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + }), + }, + { data: "[DONE]" }, + ]), + })) + await mock.module("~/lib/rate-limit", () => ({ + checkRateLimit: () => {}, + })) + const { server } = await import("~/server?route-routing") + const res = await server.request( + "/v1beta/models/gemini-pro:generateContent:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + expect(res.status).toBe(200) + const ct = res.headers.get("content-type") || "" + expect(ct.includes("text/event-stream")).toBe(true) +}) diff --git a/tests/generate-content/stream-tool-call-accumulator.test.ts b/tests/generate-content/stream-tool-call-accumulator.test.ts new file mode 100644 index 000000000..9fbde349b --- /dev/null +++ b/tests/generate-content/stream-tool-call-accumulator.test.ts @@ -0,0 +1,240 @@ +import { afterEach, expect, test, mock } from "bun:test" + +function asyncIterableFrom(events: Array<{ data?: string }>) { + return { + [Symbol.asyncIterator]() { + let i = 0 + return { + next() { + if (i < events.length) + return Promise.resolve({ value: events[i++], done: false }) + return Promise.resolve({ value: undefined, done: true }) + }, + } + }, + } +} + +afterEach(() => { + mock.restore() +}) + +test("[Stream] handles complete tool call parameters in single chunk", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => + asyncIterableFrom([ + { + data: JSON.stringify({ + id: "c1", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { + name: "ReadFile", + arguments: '{"absolute_path": "/path/to/file.txt"}', + }, + }, + ], + }, + finish_reason: null, + }, + ], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { + data: JSON.stringify({ + id: "c1", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { data: "[DONE]" }, + ]), + })) + + await mock.module("~/lib/rate-limit", () => ({ + checkRateLimit: (_: unknown) => {}, + })) + const { server } = await import("~/server?stream-complete-params") + const res = await server.request( + "/v1beta/models/gemini-pro:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "Read the file" }] }], + }), + }, + ) + + expect(res.status).toBe(200) + const body = await res.text() + expect( + body.includes( + '"functionCall":{"name":"ReadFile","args":{"absolute_path":"/path/to/file.txt"}}', + ), + ).toBe(true) +}) + +test("[Stream] handles fragmented tool call parameters across multiple chunks", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => + asyncIterableFrom([ + { + data: JSON.stringify({ + id: "c1", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { name: "ReadFile", arguments: '{"absolu' }, + }, + ], + }, + finish_reason: null, + }, + ], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { + data: JSON.stringify({ + id: "c1", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { arguments: 'te_path": "/file.txt"}' }, + }, + ], + }, + finish_reason: null, + }, + ], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { + data: JSON.stringify({ + id: "c1", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { data: "[DONE]" }, + ]), + })) + + await mock.module("~/lib/rate-limit", () => ({ + checkRateLimit: (_: unknown) => {}, + })) + const { server } = await import("~/server?stream-fragmented-params") + const res = await server.request( + "/v1beta/models/gemini-pro:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "Read the file" }] }], + }), + }, + ) + + expect(res.status).toBe(200) + const body = await res.text() + expect( + body.includes( + '"functionCall":{"name":"ReadFile","args":{"absolute_path":"/file.txt"}}', + ), + ).toBe(true) +}) + +test("[Stream] correctly processes multiple concurrent tool calls", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => + asyncIterableFrom([ + { + data: JSON.stringify({ + id: "c1", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { + name: "ReadFile", + arguments: '{"path": "/read.txt"}', + }, + }, + { + index: 1, + type: "function", + function: { + name: "WriteFile", + arguments: '{"path": "/write.txt", "content": "data"}', + }, + }, + ], + }, + finish_reason: null, + }, + ], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { + data: JSON.stringify({ + id: "c1", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { data: "[DONE]" }, + ]), + })) + + await mock.module("~/lib/rate-limit", () => ({ + checkRateLimit: (_: unknown) => {}, + })) + const { server } = await import("~/server?stream-multiple-tools") + const res = await server.request( + "/v1beta/models/gemini-pro:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "Read and write files" }] }], + }), + }, + ) + + expect(res.status).toBe(200) + const body = await res.text() + expect( + body.includes( + '"functionCall":{"name":"ReadFile","args":{"path":"/read.txt"}}', + ), + ).toBe(true) + expect( + body.includes( + '"functionCall":{"name":"WriteFile","args":{"path":"/write.txt","content":"data"}}', + ), + ).toBe(true) +}) diff --git a/tests/generate-content/streaming.test.ts b/tests/generate-content/streaming.test.ts new file mode 100644 index 000000000..d4c66770c --- /dev/null +++ b/tests/generate-content/streaming.test.ts @@ -0,0 +1,236 @@ +import { afterEach, expect, test, mock } from "bun:test" + +import type { TestServer } from "./test-types" + +import { + asyncIterableFrom, + createMockChatCompletions, + createMockRateLimit, +} from "./_test-utils" + +afterEach(() => { + mock.restore() +}) + +test("falls back to streaming when downstream returns non-stream JSON", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (_: unknown) => ({ + id: "res-3", + choices: [ + { + index: 0, + message: { role: "assistant", content: "stream me" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 2, completion_tokens: 3, total_tokens: 5 }, + }), + })) + + const { server } = (await import("~/server")) as { server: TestServer } + const res = await server.request( + "/v1beta/models/gemini-pro:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + + expect(res.status).toBe(200) + const ct = res.headers.get("content-type") || "" + expect(ct.includes("text/event-stream")).toBe(true) + const body = await res.text() + + expect(body.includes("data:")).toBe(true) + expect(body.includes("stream me")).toBe(true) + expect(body.includes('"finishReason":"STOP"')).toBe(true) + expect(body.includes('"usageMetadata"')).toBe(true) + + const occurrences = (body.match(/stream me/g) || []).length + expect(occurrences >= 1).toBe(true) +}) + +test("accumulates and parses partial JSON chunks", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (_: unknown) => { + const firstChunk = { + id: "c1", + choices: [ + { index: 0, delta: { content: "hello" }, finish_reason: null }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + const json = JSON.stringify(firstChunk) + const mid = Math.floor(json.length / 2) + const finishChunk = { + id: "c1", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + return asyncIterableFrom([ + { data: json.slice(0, mid) }, + { data: json.slice(mid) }, + { data: JSON.stringify(finishChunk) }, + { data: "[DONE]" }, + ]) + }, + })) + + await createMockRateLimit() + const { server } = (await import( + "~/server?streaming-parser-accumulation" + )) as { server: TestServer } + const res = await server.request( + "/v1beta/models/gemini-pro:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + + expect(res.status).toBe(200) + const ct = res.headers.get("content-type") || "" + expect(ct.includes("text/event-stream")).toBe(true) + const body = await res.text() + + const helloCount = (body.match(/hello/g) || []).length + expect(helloCount).toBe(1) + + expect(body.includes('"finishReason":"STOP"')).toBe(true) + expect(body.includes("data:")).toBe(true) +}) + +test("includes usageMetadata only on final chunk and injects empty part when only finish_reason", async () => { + await createMockChatCompletions([ + { + data: JSON.stringify({ + id: "c1", + choices: [ + { index: 0, delta: { content: "hello" }, finish_reason: null }, + ], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { + data: JSON.stringify({ + id: "c1", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 2, total_tokens: 3 }, + }), + }, + { data: "[DONE]" }, + ]) + + await createMockRateLimit() + const { server } = (await import( + "~/server?stream-finish-reason-and-empty-part" + )) as { server: TestServer } + const res = await server.request( + "/v1beta/models/gemini-pro:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + + expect(res.status).toBe(200) + const body = await res.text() + + const usageCount = (body.match(/"usageMetadata"/g) || []).length + expect(usageCount).toBe(1) + + const finishStop = body.includes('"finishReason":"STOP"') + expect(finishStop).toBe(true) + + const injectedEmpty = body.includes('"parts":[{"text":""}]') + expect(injectedEmpty).toBe(true) +}) + +test("[Stream] skips tool_calls with partial JSON arguments until complete", async () => { + await createMockChatCompletions([ + { + data: JSON.stringify({ + id: "c1", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { name: "f", arguments: '{"a":' }, + }, + ], + }, + finish_reason: null, + }, + ], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { + data: JSON.stringify({ + id: "c1", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + type: "function", + function: { name: "f", arguments: '{"a":1}' }, + }, + ], + }, + finish_reason: null, + }, + ], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { + data: JSON.stringify({ + id: "c1", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }, + }), + }, + { data: "[DONE]" }, + ]) + + await createMockRateLimit() + const { server } = (await import( + "~/server?stream-skip-partial-tool-calls" + )) as { + server: TestServer + } + const res = await server.request( + "/v1beta/models/gemini-pro:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + + expect(res.status).toBe(200) + const body = await res.text() + + expect(body.includes('"functionCall":{"name":"f","args"')).toBe(true) + expect(body.includes('"functionCall":{"name":"f","args":{')).toBe(true) + expect(body.includes('"functionCall":{"name":"f","args":{')).toBe(true) + expect(body.includes('"functionCall":{"name":"f","args":{"a":1}')).toBe(true) +}) diff --git a/tests/generate-content/test-types.ts b/tests/generate-content/test-types.ts new file mode 100644 index 000000000..94dddcd08 --- /dev/null +++ b/tests/generate-content/test-types.ts @@ -0,0 +1,73 @@ +import type { + ChatCompletionResponse, + ChatCompletionsPayload, +} from "~/services/copilot/create-chat-completions" + +// Test utility types +export interface TestServer { + request: ( + url: string, + options: { method: string; headers: Record; body: string }, + ) => Promise +} + +export interface MockChatCompletionsModule { + createChatCompletions: ( + payload: ChatCompletionsPayload, + ) => ChatCompletionResponse | AsyncIterable<{ data: string }> +} + +export interface MockRateLimitModule { + checkRateLimit: (payload: unknown) => void +} + +export interface MockTokenCountModule { + getTokenCount: () => { input: number; output: number } +} + +// Common test data types +export interface CapturedPayload extends Record { + messages?: Array<{ + role: string + content: string + tool_calls?: Array<{ + id: string + type: string + function: { name: string; arguments: string } + }> + }> + tools?: Array<{ + type: string + function: { name: string; parameters: Record } + }> + tool_choice?: string + model?: string +} + +// Gemini request types for tests +export interface GeminiTestRequest { + contents: Array<{ + role: "user" | "model" + parts: Array< + | { text: string } + | { functionCall: { name: string; args: Record } } + | { + functionResponse: { name: string; response: Record } + } + > + }> + tools?: Array<{ + functionDeclarations?: Array<{ + name: string + parameters: { type: string; properties?: Record } + }> + urlContext?: Record + }> + toolConfig?: { + functionCallingConfig: { mode: "AUTO" | "ANY" | "NONE" } + } + systemInstruction?: { + parts: Array<{ text: string }> + } + model?: string +} diff --git a/tests/generate-content/translation-coverage.test.ts b/tests/generate-content/translation-coverage.test.ts new file mode 100644 index 000000000..00eb6239a --- /dev/null +++ b/tests/generate-content/translation-coverage.test.ts @@ -0,0 +1,470 @@ +import { afterEach, expect, test, mock } from "bun:test" + +import type { CapturedPayload } from "./test-types" + +import { makeRequest } from "./_test-utils" + +afterEach(() => { + mock.restore() +}) + +test("processes function response arrays with tool call matching", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // Test nested function response arrays (lines 105-134) + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "Call function" }] }, + { + role: "model", + parts: [ + { + functionCall: { name: "testFunc", args: { param: "value" } }, + }, + ], + }, + { + role: "user", + parts: [ + [ + { + functionResponse: { + name: "testFunc", + response: { result: "success" }, + }, + }, + ], + ], + }, + ], + }) + + expect(res.status).toBe(200) + // This test validates that the nested array structure is processed correctly + expect(capturedPayload.messages?.length).toBeGreaterThan(0) +}) + +test("handles function response without matching tool call", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // Test case where function response has no matching tool call (line 170) + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "Call function" }] }, + { + role: "user", + parts: [ + { + functionResponse: { + name: "nonExistentFunc", + response: { result: "orphaned" }, + }, + }, + ], + }, + ], + }) + + expect(res.status).toBe(200) + const toolMessages = + capturedPayload.messages?.filter((m) => m.role === "tool") ?? [] + expect(toolMessages.length).toBe(0) // No matching tool call, so no tool message +}) + +test("handles empty content merging fallback", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // Test empty content fallback (lines 248-249) + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "" }] }, // Empty text + { role: "user", parts: [{ text: " " }] }, // Whitespace only + { role: "user", parts: [{ text: "actual question" }] }, + ], + }) + + expect(res.status).toBe(200) + const userMessages = + capturedPayload.messages?.filter((m) => m.role === "user") ?? [] + expect(userMessages.length).toBe(1) // Should merge into one message + expect(userMessages[0]?.content).toContain("actual question") +}) + +test("handles complex content that cannot be merged", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // Test complex content merging fallback (line 238) + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "First message" }] }, + { + role: "user", + parts: [ + { text: "Second message" }, + { + functionResponse: { + name: "func", + response: { data: "complex" }, + }, + }, + ], + }, + ], + }) + + expect(res.status).toBe(200) + // This test validates the content processing logic handles complex scenarios + expect(capturedPayload.messages?.length).toBeGreaterThan(0) +}) + +test("maps unsupported Gemini model names to supported ones", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // Test model mapping (lines 29-37) + const res = await makeRequest( + "/v1beta/models/gemini-2.5-flash:generateContent", + { + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }, + ) + + expect(res.status).toBe(200) + expect(capturedPayload.model).toBe("gemini-2.0-flash-001") // Should be mapped +}) + +test("preserves supported model names without mapping", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // Test model mapping preservation (line 36) + const res = await makeRequest( + "/v1beta/models/gemini-1.5-pro:generateContent", + { + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }, + ) + + expect(res.status).toBe(200) + expect(capturedPayload.model).toBe("gemini-1.5-pro") // Should remain unchanged +}) + +test("handles tool call cleanup with incomplete tool calls", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // Test incomplete tool call cleanup (lines 295-296) + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "Search something" }] }, + { + role: "model", + parts: [{ functionCall: { name: "search", args: { query: "test" } } }], + }, + // No function response - incomplete tool call that should be cleaned up + { role: "user", parts: [{ text: "What did you find?" }] }, + ], + }) + + expect(res.status).toBe(200) + // The incomplete tool call should be cleaned up + const assistantMessages = + capturedPayload.messages?.filter((m) => m.role === "assistant") ?? [] + expect(assistantMessages.length).toBe(0) // Should be cleaned up +}) + +test("processes inline data with inlineData field", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // Test inline data processing (lines 374, 377-381) + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { + role: "user", + parts: [ + { text: "Analyze this image" }, + { + inlineData: { + mimeType: "image/jpeg", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + }, + }, + ], + }, + ], + }) + + expect(res.status).toBe(200) + // This test validates inline data processing + expect(capturedPayload.messages?.length).toBeGreaterThan(0) +}) + +test("handles streaming tool calls with incomplete arguments", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => { + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // This tests the streaming tool call processing logic + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [{ role: "user", parts: [{ text: "Do a search" }] }], + }) + + expect(res.status).toBe(200) +}) + +test("accumulates streaming tool call arguments correctly", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => { + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // Test streaming arguments accumulation (lines 566-579) + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [{ role: "user", parts: [{ text: "Search for something" }] }], + }) + + expect(res.status).toBe(200) + // The request should process successfully even with complex tool call scenarios +}) + +test("handles Google Search tool processing", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // Test Google Search tool handling (lines 442-459) + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + tools: [ + { + googleSearchRetrieval: { + dynamicRetrievalConfig: { + mode: "MODE_DYNAMIC", + dynamicThreshold: 0.7, + }, + }, + }, + ], + contents: [{ role: "user", parts: [{ text: "Search for latest news" }] }], + }) + + expect(res.status).toBe(200) + // This test validates Google Search tool processing logic + expect(capturedPayload.messages?.length).toBeGreaterThan(0) +}) + +test("handles translation errors gracefully", async () => { + // Mock a scenario that would trigger error handling (lines 702-703, 881, 904) + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => { + throw new Error("Copilot API error") + }, + })) + + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [{ role: "user", parts: [{ text: "This should fail" }] }], + }) + + // Should handle the error and return appropriate status + expect(res.status).toBeGreaterThanOrEqual(400) +}) + +test("handles malformed tool calls in content processing", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => { + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // Test malformed function call handling + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "Process this" }] }, + { + role: "model", + parts: [ + { + functionCall: { + name: "", // Empty name should trigger error handling + args: {}, + }, + }, + ], + }, + ], + }) + + expect(res.status).toBe(200) + // Should handle malformed calls gracefully +}) diff --git a/tests/generate-content/translation-response-coverage.test.ts b/tests/generate-content/translation-response-coverage.test.ts new file mode 100644 index 000000000..78873696c --- /dev/null +++ b/tests/generate-content/translation-response-coverage.test.ts @@ -0,0 +1,132 @@ +import { describe, it, expect } from "bun:test" + +import type { ChatCompletionResponse } from "~/services/copilot/create-chat-completions" + +import { translateOpenAIToGemini } from "~/routes/generate-content/translation" + +describe("OpenAI to Gemini Response Translation", () => { + it("should handle assistant message with tool calls having arguments", () => { + const openAIResponse: ChatCompletionResponse = { + id: "chatcmpl-123", + object: "chat.completion", + created: Date.now(), + model: "gpt-4", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: "I'll search for that", + tool_calls: [ + { + id: "call_123", + type: "function", + function: { + name: "search", + arguments: '{"query": "test query", "limit": 10}', + }, + }, + ], + }, + finish_reason: "tool_calls", + logprobs: null, + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 20, + total_tokens: 30, + }, + } + + const result = translateOpenAIToGemini(openAIResponse) + + expect(result.candidates).toHaveLength(1) + expect(result.candidates[0]?.content.parts).toHaveLength(2) + expect(result.candidates[0]?.content.parts[0]).toEqual({ + text: "I'll search for that", + }) + expect(result.candidates[0]?.content.parts[1]).toEqual({ + functionCall: { + name: "search", + args: { query: "test query", limit: 10 }, + }, + }) + }) + + it("should handle assistant message with tool calls having empty arguments", () => { + const openAIResponse: ChatCompletionResponse = { + id: "chatcmpl-456", + object: "chat.completion", + created: Date.now(), + model: "gpt-4", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: "Getting current time", + tool_calls: [ + { + id: "call_456", + type: "function", + function: { + name: "get_current_time", + arguments: "", + }, + }, + ], + }, + finish_reason: "tool_calls", + logprobs: null, + }, + ], + usage: { + prompt_tokens: 5, + completion_tokens: 10, + total_tokens: 15, + }, + } + + const result = translateOpenAIToGemini(openAIResponse) + + expect(result.candidates[0]?.content.parts[1]).toEqual({ + functionCall: { + name: "get_current_time", + args: {}, + }, + }) + }) + + it("should handle assistant message with simple text content", () => { + const openAIResponse: ChatCompletionResponse = { + id: "chatcmpl-789", + object: "chat.completion", + created: Date.now(), + model: "gpt-4", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: "Here's my response", + }, + finish_reason: "stop", + logprobs: null, + }, + ], + usage: { + prompt_tokens: 15, + completion_tokens: 5, + total_tokens: 20, + }, + } + + const result = translateOpenAIToGemini(openAIResponse) + + expect(result.candidates[0]?.content.parts).toHaveLength(1) + expect(result.candidates[0]?.content.parts[0]).toEqual({ + text: "Here's my response", + }) + }) +}) diff --git a/tests/generate-content/translation.test.ts b/tests/generate-content/translation.test.ts new file mode 100644 index 000000000..559e25146 --- /dev/null +++ b/tests/generate-content/translation.test.ts @@ -0,0 +1,320 @@ +import { afterEach, expect, test, mock } from "bun:test" + +import type { CapturedPayload } from "./test-types" + +import { makeRequest } from "./_test-utils" + +afterEach(() => { + mock.restore() +}) + +test("processes toolConfig AUTO/ANY/NONE mapping end-to-end", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + // toolConfig 需要有 tools 才会被处理，所以添加 tools + const baseRequest = { + tools: [ + { + functionDeclarations: [ + { name: "test", parameters: { type: "object" } }, + ], + }, + ], + contents: [{ role: "user", parts: [{ text: "hi" }] }], + } + + // Test AUTO -> auto + const autoRes = await makeRequest( + "/v1beta/models/gemini-pro:generateContent", + { + ...baseRequest, + toolConfig: { functionCallingConfig: { mode: "AUTO" } }, + }, + ) + expect(autoRes.status).toBe(200) + expect(capturedPayload.tool_choice).toBe("auto") + + // Test ANY -> required + const anyRes = await makeRequest( + "/v1beta/models/gemini-pro:generateContent", + { + ...baseRequest, + toolConfig: { functionCallingConfig: { mode: "ANY" } }, + }, + ) + expect(anyRes.status).toBe(200) + expect(capturedPayload.tool_choice).toBe("required") + + // Test NONE -> none + const noneRes = await makeRequest( + "/v1beta/models/gemini-pro:generateContent", + { + ...baseRequest, + toolConfig: { functionCallingConfig: { mode: "NONE" } }, + }, + ) + expect(noneRes.status).toBe(200) + expect(capturedPayload.tool_choice).toBe("none") +}) + +test("handles urlContext tool filtering in request", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + tools: [ + { urlContext: {} }, + { + functionDeclarations: [ + { name: "readFile", parameters: { type: "object" } }, + ], + }, + ], + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }) + + expect(res.status).toBe(200) + expect(capturedPayload.tools).toBeDefined() + const toolNames = new Set( + capturedPayload.tools?.map((t) => t.function.name) ?? [], + ) + expect(toolNames.has("readFile")).toBe(true) + expect(toolNames.has("urlContext")).toBe(false) +}) + +test("synthesizes tools from function calls when tools not provided", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "Do a web search" }] }, + { + role: "model", + parts: [{ functionCall: { name: "search", args: { query: "cats" } } }], + }, + ], + }) + + expect(res.status).toBe(200) + expect(capturedPayload.tools).toBeDefined() + const toolNames = capturedPayload.tools?.map((t) => t.function.name) ?? [] + expect(toolNames.includes("search")).toBe(true) +}) + +test("handles same-role message merging behavior", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "Hello." }] }, + { role: "user", parts: [{ text: "How are you?" }] }, + ], + }) + + expect(res.status).toBe(200) + const userMessages = + capturedPayload.messages?.filter((m) => m.role === "user") ?? [] + expect(userMessages.length).toBe(1) + expect(userMessages[0]?.content).toContain("Hello.") + expect(userMessages[0]?.content).toContain("How are you?") +}) + +test("handles incomplete tool calls cleanup", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "Search for cats." }] }, + { + role: "model", + parts: [{ functionCall: { name: "search", args: { query: "cats" } } }], + }, + { role: "user", parts: [{ text: "Show me results." }] }, + ], + }) + + expect(res.status).toBe(200) + const assistantMessages = + capturedPayload.messages?.filter((m) => m.role === "assistant") ?? [] + expect(assistantMessages.length).toBe(0) + const userMessages = + capturedPayload.messages?.filter((m) => m.role === "user") ?? [] + expect(userMessages.length).toBeGreaterThan(0) +}) + +test("handles system instruction in contents", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + systemInstruction: { parts: [{ text: "You are a helpful assistant" }] }, + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }) + + expect(res.status).toBe(200) + const systemMessage = capturedPayload.messages?.find( + (m) => m.role === "system", + ) + expect(systemMessage).toBeDefined() + expect(systemMessage?.content).toContain("helpful assistant") +}) + +test("handles empty contents gracefully", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => { + throw new Error("Should not be called with empty contents") + }, + })) + + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [], + }) + + // 空 contents 会导致翻译过程中出错，返回 500 + expect(res.status).toBe(500) +}) + +test("handles complex tool call workflow", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } + }, + })) + + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "Read a file" }] }, + { + role: "model", + parts: [ + { functionCall: { name: "readFile", args: { path: "test.txt" } } }, + ], + }, + { + role: "user", + parts: [ + { + functionResponse: { + name: "readFile", + response: { content: "Hello World" }, + }, + }, + ], + }, + ], + }) + + expect(res.status).toBe(200) + expect( + capturedPayload.messages?.some( + (m) => m.role === "assistant" && m.tool_calls, + ), + ).toBe(true) + expect(capturedPayload.messages?.some((m) => m.role === "tool")).toBe(true) +}) diff --git a/tests/generate-content/validation-and-routing.test.ts b/tests/generate-content/validation-and-routing.test.ts new file mode 100644 index 000000000..d391e60f4 --- /dev/null +++ b/tests/generate-content/validation-and-routing.test.ts @@ -0,0 +1,236 @@ +import { afterEach, expect, test, mock } from "bun:test" + +import { + asyncIterableFrom, + createMockRateLimit, + makeRequest, +} from "./_test-utils" + +afterEach(() => { + mock.restore() +}) + +test("forwards generic errors as HTTP 500", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => { + throw new Error("Internal issue") + }, + })) + const { server } = await import("~/server") + const res = await server.request( + "/v1beta/models/gemini-pro:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + expect(res.status).toBe(500) + const json = (await res.json()) as { + error: { message: string; type: string } + } + expect(json).toEqual({ error: { message: "Internal issue", type: "error" } }) +}) + +test("requires model in URL for non-stream endpoint", async () => { + const { server } = await import("~/server") + const res = await server.request("/v1beta/models/:generateContent", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }) + + expect(res.status).toBe(500) + const json = await res.json() + expect(json).toEqual({ + error: { message: "Model name is required in URL path", type: "error" }, + }) +}) + +test("requires model in URL for stream endpoint", async () => { + const { server } = await import("~/server") + const res = await server.request("/v1beta/models/:streamGenerateContent", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }) + + expect(res.status).toBe(500) + const json = await res.json() + expect(json).toEqual({ + error: { message: "Model name is required in URL path", type: "error" }, + }) +}) + +test("requires model in URL for countTokens endpoint", async () => { + const { server } = await import("~/server") + const res = await server.request("/v1beta/models/:countTokens", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }) + + expect(res.status).toBe(500) + const json = await res.json() + expect(json).toEqual({ + error: { message: "Model name is required in URL path", type: "error" }, + }) +}) + +test("streams fallback response when no text content in non-streaming to streaming conversion", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (_: unknown) => ({ + id: "res-fallback", + choices: [ + { + index: 0, + message: { role: "assistant", content: null }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 0, total_tokens: 1 }, + }), + })) + + await createMockRateLimit() + + const { server } = await import("~/server?fallback-response-no-text") + + const res = await server.request( + "/v1beta/models/gemini-pro:streamGenerateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "test" }] }], + }), + }, + ) + + expect(res.status).toBe(200) + const ct = res.headers.get("content-type") || "" + expect(ct.includes("text/event-stream")).toBe(true) + + const body = await res.text() + + expect(body.includes("data:")).toBe(true) + expect(body.includes('"candidates"')).toBe(true) + expect(body.includes('"usageMetadata"')).toBe(true) +}) + +test("non-stream endpoint rejects streaming response with 500", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (_: unknown) => + asyncIterableFrom([ + { + data: JSON.stringify({ + id: "c1", + choices: [ + { index: 0, delta: { content: "x" }, finish_reason: null }, + ], + }), + }, + { + data: JSON.stringify({ + id: "c1", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + }), + }, + { data: "[DONE]" }, + ]), + })) + + const { server } = await import("~/server") + const res = await server.request( + "/v1beta/models/gemini-pro:generateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }), + }, + ) + + expect(res.status).toBe(500) + const json = await res.json() + expect(json).toEqual({ + error: { + message: "Unexpected streaming response for non-streaming endpoint", + type: "error", + }, + }) +}) + +test("routes fallthrough when URL doesn't match any generate-content patterns", async () => { + await createMockRateLimit() + + const { server } = await import("~/server?route-fallthrough") + + const res = await server.request( + "/v1beta/models/gemini-pro:unknownOperation", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "test" }] }], + }), + }, + ) + + expect(res.status).toBe(404) +}) + +test("handles HTTP errors with proper error codes", async () => { + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: () => { + const error = new Error("Bad Request") + // Simulate HTTPError-like structure + Object.assign(error, { status: 400 }) + throw error + }, + })) + + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [{ role: "user", parts: [{ text: "hi" }] }], + }) + + // 由于错误处理机制，HTTP错误也会转为500 + expect(res.status).toBe(500) + const json = (await res.json()) as { + error: { message: string; type: string } + } + expect(json.error.message).toContain("Bad Request") +}) + +test("handles malformed JSON in request body", async () => { + const { server } = await import("~/server") + const res = await server.request( + "/v1beta/models/gemini-pro:generateContent", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: "{ invalid json", + }, + ) + + // JSON 解析错误会返回 500 + expect(res.status).toBe(500) +}) + +test("validates required contents field in request", async () => { + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + // Missing contents field + model: "gemini-pro", + }) + + expect([400, 500]).toContain(res.status) +}) From faf03e6e6bea33d2452d22303b8cd88b77ae2fa2 Mon Sep 17 00:00:00 2001 From: cpf Date: Sat, 27 Sep 2025 00:59:29 +0800 Subject: [PATCH 07/16] trigger --- src/routes/generate-content/translation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/generate-content/translation.ts b/src/routes/generate-content/translation.ts index c3ea83a81..cc2b4a8e6 100644 --- a/src/routes/generate-content/translation.ts +++ b/src/routes/generate-content/translation.ts @@ -536,7 +536,7 @@ export function translateOpenAIToGemini( }, } - // Debug: Log original GitHub Copilot response and translated Gemini response for comparison + // Debug: Log original GitHub Copilot response and translated Gemini response if (process.env.DEBUG_GEMINI_REQUESTS === "true") { DebugLogger.logResponseComparison(response, result, { context: "Non-Stream Response Translation", From 8f21b6e696467bc20b5bce60037501392961ceb9 Mon Sep 17 00:00:00 2001 From: cpf Date: Sat, 27 Sep 2025 01:11:30 +0800 Subject: [PATCH 08/16] fix: update server import for fallback non-streaming in streaming tests --- tests/generate-content/streaming.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/generate-content/streaming.test.ts b/tests/generate-content/streaming.test.ts index d4c66770c..b55030175 100644 --- a/tests/generate-content/streaming.test.ts +++ b/tests/generate-content/streaming.test.ts @@ -27,7 +27,10 @@ test("falls back to streaming when downstream returns non-stream JSON", async () }), })) - const { server } = (await import("~/server")) as { server: TestServer } + await createMockRateLimit() + const { server } = (await import("~/server?fallback-non-streaming")) as { + server: TestServer + } const res = await server.request( "/v1beta/models/gemini-pro:streamGenerateContent", { From 1beeb720f5cbd138b893ccb52ed08dcf442007a5 Mon Sep 17 00:00:00 2001 From: cpf Date: Wed, 1 Oct 2025 17:33:39 +0800 Subject: [PATCH 09/16] feat: refactor tool translation and streaming handling for improved functionality --- src/lib/tool-call-utils.ts | 312 ++++++++++++ src/routes/generate-content/handler.ts | 189 +++----- src/routes/generate-content/translation.ts | 539 ++++++--------------- 3 files changed, 544 insertions(+), 496 deletions(-) create mode 100644 src/lib/tool-call-utils.ts diff --git a/src/lib/tool-call-utils.ts b/src/lib/tool-call-utils.ts new file mode 100644 index 000000000..f642b5e9d --- /dev/null +++ b/src/lib/tool-call-utils.ts @@ -0,0 +1,312 @@ +import type { + GeminiTool, + GeminiRequest, + GeminiContent, + GeminiPart, +} from "~/routes/generate-content/types" +import type { Tool, Message } from "~/services/copilot/create-chat-completions" + +// Tool declaration generation - moved from translation.ts +export function translateGeminiToolsToOpenAI( + geminiTools?: Array, +): Array | undefined { + if (!geminiTools || geminiTools.length === 0) return undefined + + const tools: Array = [] + for (const tool of geminiTools) { + // Handle standard function declarations + if (tool.functionDeclarations) { + for (const func of tool.functionDeclarations) { + // Validate that function name exists and is not empty + if ( + !func.name + || typeof func.name !== "string" + || func.name.trim() === "" + ) { + continue + } + + // Ensure parameters is always a valid object + const validParameters = func.parametersJsonSchema + || func.parameters || { type: "object", properties: {} } + + tools.push({ + type: "function", + function: { + name: func.name, + description: func.description, + parameters: validParameters, + }, + }) + } + } + + // Handle googleSearch tool (special case) + if (tool.googleSearch !== undefined) { + tools.push({ + type: "function", + function: { + name: "google_web_search", + description: + "Performs a web search using Google Search (via the Gemini API) and returns the results. This tool is useful for finding information on the internet based on a query.", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "The search query to find information on the web.", + }, + }, + required: ["query"], + }, + }, + }) + } + + // Handle urlContext tool (special case for web_fetch) + // Note: GitHub Copilot API doesn't support web_fetch functionality + // Skip this tool to avoid "Failed to create chat completions" errors + if (tool.urlContext !== undefined) { + continue + } + } + + return tools.length > 0 ? tools : undefined +} + +// Tool configuration translation - moved from translation.ts +export function translateGeminiToolConfigToOpenAI( + toolConfig?: GeminiRequest["toolConfig"], +): "auto" | "required" | "none" | undefined { + if (!toolConfig) return undefined + + const mode = toolConfig.functionCallingConfig.mode + switch (mode) { + case "AUTO": { + return "auto" + } + case "ANY": { + return "required" + } + case "NONE": { + return "none" + } + default: { + return undefined + } + } +} + +// Tool response deduplication - moved from translation.ts +export function ensureToolCallResponseMatch( + messages: Array, +): Array { + const result: Array = [] + const seenToolCallIds = new Set() // Track processed tool_call_ids to avoid duplicates + + for (const message of messages) { + if (message.role === "tool" && message.tool_call_id) { + const toolCallId = message.tool_call_id + + // Only keep the FIRST response for each tool_call_id (deduplicate) + if (!seenToolCallIds.has(toolCallId)) { + seenToolCallIds.add(toolCallId) + result.push(message) + } + // Skip any duplicate responses for the same tool_call_id + } else { + // Keep all non-tool messages as-is + result.push(message) + } + } + + return result +} + +// Utility function to generate unique tool call IDs - moved from translation.ts +// Generate IDs within 40 character limit (API constraint) +export function generateToolCallId(_functionName: string): string { + const timestamp = Date.now().toString(36) // Base36 for shorter encoding + const random = Math.random().toString(36).slice(2, 8) // 6 chars random + return `call_${timestamp}_${random}` // Format: call_{timestamp}_{random} +} + +// Helper function to try parsing and creating a function call - moved from translation.ts +export function tryCreateFunctionCall( + name: string, + argumentsStr: string, +): GeminiPart | null { + try { + const args = JSON.parse(argumentsStr) as Record + return { + functionCall: { + name, + args, + }, + } + } catch { + return null + } +} + +// Tool synthesis from contents - moved from translation.ts +export function synthesizeToolsFromContents( + contents: Array< + | GeminiContent + | Array<{ + functionResponse: { id?: string; name: string; response: unknown } + }> + >, +): Array | undefined { + const names = new Set() + for (const item of contents) { + if (Array.isArray(item)) continue + for (const part of item.parts) { + if ("functionCall" in part && part.functionCall.name) { + names.add(part.functionCall.name) + } + } + } + if (names.size === 0) return undefined + return Array.from(names).map((name) => ({ + type: "function", + function: { name, parameters: { type: "object", properties: {} } }, + })) +} + +/** + * 工具调用状态管理器 + * 用于流式响应中工具调用参数的增量累积 + */ +export class ToolCallAccumulator { + private accumulator = new Map< + number, + { + name: string + arguments: string + id?: string + } + >() + + /** + * 处理带有函数名的工具调用（新工具调用的开始） + */ + handleToolCallWithName(toolCall: { + index: number + id?: string + function: { + name: string + arguments?: string + } + }): GeminiPart | null { + const accumulatedArgs = toolCall.function.arguments || "" + + this.accumulator.set(toolCall.index, { + name: toolCall.function.name, + arguments: accumulatedArgs, + id: toolCall.id, + }) + + // If we already have arguments, try to process immediately (for non-streaming models like Gemini) + if (accumulatedArgs) { + const functionCall = tryCreateFunctionCall( + toolCall.function.name, + accumulatedArgs, + ) + if (functionCall) { + // Clear the accumulator for this index since we've successfully processed it + this.accumulator.delete(toolCall.index) + return functionCall + } + } + + return null + } + + /** + * 处理工具调用参数累积（追加参数片段） + */ + handleToolCallAccumulation(toolCall: { + index: number + function?: { + arguments?: string + } + }): GeminiPart | null { + const existingAccumulated = this.accumulator.get(toolCall.index) + + if (existingAccumulated && toolCall.function?.arguments) { + existingAccumulated.arguments += toolCall.function.arguments + + const functionCall = tryCreateFunctionCall( + existingAccumulated.name, + existingAccumulated.arguments, + ) + if (functionCall) { + // Clear the accumulator for this index since we've successfully processed it + this.accumulator.delete(toolCall.index) + return functionCall + } + } + + return null + } + + /** + * 清理所有累积状态（用于流结束或错误重置） + */ + clear(): void { + this.accumulator.clear() + } +} + +/** + * 处理工具调用数组，生成 Gemini 格式的部分 + * 支持完整参数和分片参数两种模式 + */ +export function processToolCalls( + toolCalls: Array<{ + index: number + id?: string + type?: "function" + function?: { + name?: string + arguments?: string + } + }>, + accumulator: ToolCallAccumulator, +): Array { + const parts: Array = [] + + for (const toolCall of toolCalls) { + // Debug: Log streaming tool call arguments to verify what GitHub Copilot returns + if (process.env.DEBUG_GEMINI_REQUESTS === "true") { + console.log( + `[DEBUG STREAM] Tool call - name: ${toolCall.function?.name}, arguments: "${toolCall.function?.arguments}", type: ${typeof toolCall.function?.arguments}, truthy: ${Boolean(toolCall.function?.arguments)}`, + ) + } + + // If this chunk has a function name, it's the start of a new tool call + if (toolCall.function?.name && toolCall.function.name.trim() !== "") { + const functionCall = accumulator.handleToolCallWithName({ + index: toolCall.index, + id: toolCall.id, + function: { + name: toolCall.function.name, + arguments: toolCall.function.arguments, + }, + }) + if (functionCall) { + parts.push(functionCall) + } + continue + } + + // If we have existing accumulated data and this chunk has arguments, append them + const functionCall = accumulator.handleToolCallAccumulation(toolCall) + if (functionCall) { + parts.push(functionCall) + } + } + + return parts +} diff --git a/src/routes/generate-content/handler.ts b/src/routes/generate-content/handler.ts index af0f2d3fd..88ec39473 100644 --- a/src/routes/generate-content/handler.ts +++ b/src/routes/generate-content/handler.ts @@ -23,9 +23,10 @@ function extractModelFromUrl(url: string): string { return match[1] } +import { ToolCallAccumulator } from "~/lib/tool-call-utils" + import { - translateGeminiToOpenAINonStream, - translateGeminiToOpenAIStream, + translateGeminiToOpenAI, translateOpenAIToGemini, translateGeminiCountTokensToOpenAI, translateTokenCountToGemini, @@ -38,8 +39,11 @@ import { type GeminiResponse, } from "./types" -// Standard generation endpoint -export async function handleGeminiGeneration(c: Context) { +// Unified generation handler following Claude's two-branch pattern +export async function handleGeminiGeneration( + c: Context, + stream: boolean = false, +) { const model = extractModelFromUrl(c.req.url) if (!model) { @@ -49,8 +53,16 @@ export async function handleGeminiGeneration(c: Context) { await checkRateLimit(state) const geminiPayload = await c.req.json() + const openAIPayload = translateGeminiToOpenAI(geminiPayload, model, stream) - const openAIPayload = translateGeminiToOpenAINonStream(geminiPayload, model) + // Log request for debugging (async, non-blocking) - only if debug logging is enabled + if (process.env.DEBUG_GEMINI_REQUESTS === "true") { + DebugLogger.logGeminiRequest(geminiPayload, openAIPayload).catch( + (error: unknown) => { + console.error("[DEBUG] Failed to log request:", error) + }, + ) + } if (state.manualApprove) { await awaitApproval() @@ -61,11 +73,17 @@ export async function handleGeminiGeneration(c: Context) { if (isNonStreaming(response)) { const geminiResponse = translateOpenAIToGemini(response) + if (stream) { + return handleNonStreamingToStreaming(c, geminiResponse) + } return c.json(geminiResponse) } - // This shouldn't happen for non-streaming endpoint - throw new Error("Unexpected streaming response for non-streaming endpoint") + if (!stream) { + throw new Error("Unexpected streaming response for non-streaming endpoint") + } + + return handleStreamingResponse(c, response) } // Helper function to handle non-streaming response conversion @@ -156,29 +174,36 @@ async function sendFallbackResponse( await stream.writeSSE({ data: JSON.stringify(streamResponse) }) } -// Accumulative JSON parser for handling incomplete chunks (based on LiteLLM research) -class StreamingJSONParser { - private accumulatedData = "" - private parseMode: "direct" | "accumulated" = "direct" +// Simplified Gemini streaming state (inspired by Claude AnthropicStreamState) +interface GeminiStreamState { + jsonAccumulator: string + parseMode: "direct" | "accumulated" +} + +// Minimal state machine for JSON parsing only +class GeminiStreamParser { + private state: GeminiStreamState = { + jsonAccumulator: "", + parseMode: "direct", + } parseChunk(rawData: string): unknown { - if (this.parseMode === "direct") { + if (this.state.parseMode === "direct") { try { return JSON.parse(rawData) } catch { - // Switch to accumulated mode on first failure (LiteLLM pattern) - this.parseMode = "accumulated" - this.accumulatedData = rawData + // Switch to accumulated mode on first failure + this.state.parseMode = "accumulated" + this.state.jsonAccumulator = rawData return null } } else { // Accumulated mode - keep building until valid JSON - this.accumulatedData += rawData + this.state.jsonAccumulator += rawData try { - const result = JSON.parse(this.accumulatedData) as unknown + const result = JSON.parse(this.state.jsonAccumulator) as unknown // Success - reset for next chunk - this.accumulatedData = "" - this.parseMode = "direct" // Can switch back to direct mode + this.resetAccumulator() return result } catch { // Continue accumulating @@ -186,55 +211,10 @@ class StreamingJSONParser { } } } -} - -// Global parser instance for the stream -// let streamParser = new StreamingJSONParser() -// Helper function to process chunk and write to stream -async function processAndWriteChunk(params: { - rawEvent: { data?: string } - stream: SSEStreamingApi - lastWritePromise: Promise - streamParser: StreamingJSONParser -}): Promise<{ newWritePromise: Promise; hasFinishReason: boolean }> { - const { rawEvent, stream, lastWritePromise, streamParser } = params - - if (!rawEvent.data) { - return { newWritePromise: lastWritePromise, hasFinishReason: false } - } - - try { - const chunk = streamParser.parseChunk(rawEvent.data) - - // If parser returns null, we're still accumulating - if (!chunk) { - return { newWritePromise: lastWritePromise, hasFinishReason: false } - } - - const geminiChunk = translateOpenAIChunkToGemini( - chunk as ChatCompletionChunk, - ) - - if (geminiChunk) { - // Check if this chunk contains a finish reason - const chunkHasFinishReason = geminiChunk.candidates.some( - (c) => c.finishReason && c.finishReason !== "FINISH_REASON_UNSPECIFIED", - ) - - // Wait for previous write to complete before writing new chunk - await lastWritePromise - const newWritePromise = stream.writeSSE({ - data: JSON.stringify(geminiChunk), - }) - - return { newWritePromise, hasFinishReason: chunkHasFinishReason } - } else { - return { newWritePromise: lastWritePromise, hasFinishReason: false } - } - } catch (parseError) { - console.error("[GEMINI_STREAM] Error parsing chunk", parseError) - return { newWritePromise: lastWritePromise, hasFinishReason: false } + private resetAccumulator(): void { + this.state.jsonAccumulator = "" + this.state.parseMode = "direct" } } @@ -245,7 +225,9 @@ function handleStreamingResponse( ) { return streamSSE(c, async (stream) => { // Create a parser instance for this stream (each request gets its own parser) - const streamParser = new StreamingJSONParser() + const streamParser = new GeminiStreamParser() + // Create a tool call accumulator for this stream + const toolCallAccumulator = new ToolCallAccumulator() let lastWritePromise: Promise = Promise.resolve() try { @@ -254,13 +236,32 @@ function handleStreamingResponse( break } - const result = await processAndWriteChunk({ - rawEvent, - stream, - lastWritePromise, - streamParser, - }) - lastWritePromise = result.newWritePromise + // Inline processing without extra wrapper + if (!rawEvent.data) { + continue + } + + try { + const chunk = streamParser.parseChunk(rawEvent.data) + if (!chunk) { + continue + } + + const geminiChunk = translateOpenAIChunkToGemini( + chunk as ChatCompletionChunk, + toolCallAccumulator, + ) + if (geminiChunk) { + // Wait for previous write to complete before writing new chunk + await lastWritePromise + lastWritePromise = stream.writeSSE({ + data: JSON.stringify(geminiChunk), + }) + } + } catch (parseError) { + console.error("[GEMINI_STREAM] Error parsing chunk", parseError) + continue + } } // Wait for all writes to complete before closing @@ -282,41 +283,9 @@ function handleStreamingResponse( }) } -// Streaming generation endpoint -export async function handleGeminiStreamGeneration(c: Context) { - const model = extractModelFromUrl(c.req.url) - - if (!model) { - throw new Error("Model name is required in URL path") - } - - await checkRateLimit(state) - - const geminiPayload = await c.req.json() - - const openAIPayload = translateGeminiToOpenAIStream(geminiPayload, model) - - // Log request for debugging (async, non-blocking) - only if debug logging is enabled - if (process.env.DEBUG_GEMINI_REQUESTS === "true") { - DebugLogger.logGeminiRequest(geminiPayload, openAIPayload).catch( - (error: unknown) => { - console.error("[DEBUG] Failed to log request:", error) - }, - ) - } - if (state.manualApprove) { - await awaitApproval() - } - - const response = await createChatCompletions(openAIPayload) - - if (isNonStreaming(response)) { - const geminiResponse = translateOpenAIToGemini(response) - - return handleNonStreamingToStreaming(c, geminiResponse) - } - - return handleStreamingResponse(c, response) +// Create convenience wrapper for streaming generation +export function handleGeminiStreamGeneration(c: Context) { + return handleGeminiGeneration(c, true) } // Token counting endpoint diff --git a/src/routes/generate-content/translation.ts b/src/routes/generate-content/translation.ts index cc2b4a8e6..8f4d70937 100644 --- a/src/routes/generate-content/translation.ts +++ b/src/routes/generate-content/translation.ts @@ -1,4 +1,12 @@ import { DebugLogger } from "~/lib/debug-logger" +import { + translateGeminiToolsToOpenAI, + translateGeminiToolConfigToOpenAI, + generateToolCallId, + synthesizeToolsFromContents, + ToolCallAccumulator, + processToolCalls as processToolCallsWithAccumulator, +} from "~/lib/tool-call-utils" import { type ChatCompletionResponse, type ChatCompletionChunk, @@ -53,33 +61,10 @@ function selectTools( // Request translation: Gemini -> OpenAI -export function translateGeminiToOpenAINonStream( - payload: GeminiRequest, - model: string, -): ChatCompletionsPayload { - const tools = selectTools(payload.tools, payload.contents) - const result = { - model: mapGeminiModelToCopilot(model), - messages: translateGeminiContentsToOpenAI( - payload.contents, - payload.systemInstruction, - ), - max_tokens: (payload.generationConfig?.maxOutputTokens as number) || 4096, - stop: payload.generationConfig?.stopSequences as Array | undefined, - stream: false, - temperature: payload.generationConfig?.temperature as number | undefined, - top_p: payload.generationConfig?.topP as number | undefined, - tools, - tool_choice: - tools ? translateGeminiToolConfigToOpenAI(payload.toolConfig) : undefined, - } - - return result -} - -export function translateGeminiToOpenAIStream( +export function translateGeminiToOpenAI( payload: GeminiRequest, model: string, + stream: boolean, ): ChatCompletionsPayload { const tools = selectTools(payload.tools, payload.contents) const result = { @@ -90,7 +75,7 @@ export function translateGeminiToOpenAIStream( ), max_tokens: (payload.generationConfig?.maxOutputTokens as number) || 4096, stop: payload.generationConfig?.stopSequences as Array | undefined, - stream: true, + stream, temperature: payload.generationConfig?.temperature as number | undefined, top_p: payload.generationConfig?.topP as number | undefined, tools, @@ -213,59 +198,112 @@ function processFunctionCalls(options: { }) } -// Helper function to merge consecutive messages with same role -function mergeConsecutiveSameRoleMessages( +// Helper function to check if a tool response is duplicate +function isDuplicateToolResponse( + message: Message, + seenToolCallIds: Set, +): boolean { + return ( + message.role === "tool" + && message.tool_call_id !== undefined + && seenToolCallIds.has(message.tool_call_id) + ) +} + +// Helper function to normalize user message content +function normalizeUserMessageContent(message: Message): void { + if ( + message.role === "user" + && typeof message.content === "string" + && !message.content.trim() + ) { + message.content = " " // Add minimal text content as fallback + } +} + +// Helper function to check if messages can be merged +function canMergeMessages( + lastMessage: Message, + currentMessage: Message, +): boolean { + return ( + lastMessage.role === currentMessage.role + && !lastMessage.tool_calls + && !currentMessage.tool_calls + && !(lastMessage as { tool_call_id?: string }).tool_call_id + && !(currentMessage as { tool_call_id?: string }).tool_call_id + && typeof lastMessage.content === "string" + && typeof currentMessage.content === "string" + ) +} + +// Helper function to check if message should be skipped +function shouldSkipMessage( + message: Message, messages: Array, -): Array { - const mergedMessages: Array = [] - for (const message of messages) { - const lastMessage = mergedMessages.at(-1) + seenToolCallIds: Set, +): boolean { + // Skip incomplete assistant messages with tool calls that have no responses + if ( + message.role === "assistant" + && message.tool_calls + && !hasCorrespondingToolResponses(messages, message.tool_calls) + ) { + return true + } + + // Skip duplicate tool responses + if (isDuplicateToolResponse(message, seenToolCallIds)) { + return true + } + + return false +} +// Helper function to process and add message to cleaned array +function processAndAddMessage( + message: Message, + cleanedMessages: Array, + seenToolCallIds: Set, +): void { + // Track tool call IDs for deduplication + if (message.role === "tool" && message.tool_call_id) { + seenToolCallIds.add(message.tool_call_id) + } + + // Normalize user message content + normalizeUserMessageContent(message) + + // Try to merge with previous message + const lastMessage = cleanedMessages.at(-1) + if (lastMessage && canMergeMessages(lastMessage, message)) { + // Merge with previous message of same role + // canMergeMessages already ensures both contents are strings if ( - lastMessage - && lastMessage.role === message.role - && !lastMessage.tool_calls - && !message.tool_calls - && !(lastMessage as { tool_call_id?: string }).tool_call_id // Don't merge tool responses - && !(message as { tool_call_id?: string }).tool_call_id // Don't merge tool responses + typeof lastMessage.content === "string" + && typeof message.content === "string" ) { - // Merge with previous message of same role - if ( - typeof lastMessage.content === "string" - && typeof message.content === "string" - ) { - lastMessage.content = lastMessage.content + "\n\n" + message.content - } else { - // Can't merge complex content, keep separate - mergedMessages.push(message) - } - } else { - // Add content validation for user messages (based on LiteLLM research) - if ( - message.role === "user" - && typeof message.content === "string" - && !message.content.trim() - ) { - message.content = " " // Add minimal text content as fallback - } - mergedMessages.push(message) + lastMessage.content = `${lastMessage.content}\n\n${message.content}` } + } else { + cleanedMessages.push(message) } - return mergedMessages } -// Helper function to remove incomplete assistant messages -function removeIncompleteAssistantMessages(messages: Array): void { - for (let i = messages.length - 1; i >= 0; i--) { - const message = messages[i] - if ( - message.role === "assistant" - && message.tool_calls - && !hasCorrespondingToolResponses(messages, message.tool_calls) - ) { - messages.splice(i, 1) +// Consolidated message cleanup function +function cleanupMessages(messages: Array): Array { + const cleanedMessages: Array = [] + const seenToolCallIds = new Set() + + for (const message of messages) { + if (shouldSkipMessage(message, messages, seenToolCallIds)) { + continue } + + processAndAddMessage(message, cleanedMessages, seenToolCallIds) } + + return cleanedMessages } function translateGeminiContentsToOpenAI( @@ -327,38 +365,8 @@ function translateGeminiContentsToOpenAI( } } - // Post-process: Remove incomplete assistant messages from cancelled tool calls - removeIncompleteAssistantMessages(messages) - - // Post-process: Deduplicate tool responses (remove duplicate tool_call_ids) - const matchedMessages = ensureToolCallResponseMatch(messages) - - // Post-process: Merge consecutive messages with same role (based on LiteLLM research) - return mergeConsecutiveSameRoleMessages(matchedMessages) -} - -function synthesizeToolsFromContents( - contents: Array< - | GeminiContent - | Array<{ - functionResponse: { id?: string; name: string; response: unknown } - }> - >, -): Array | undefined { - const names = new Set() - for (const item of contents) { - if (Array.isArray(item)) continue - for (const part of item.parts) { - if ("functionCall" in part && part.functionCall.name) { - names.add(part.functionCall.name) - } - } - } - if (names.size === 0) return undefined - return Array.from(names).map((name) => ({ - type: "function", - function: { name, parameters: { type: "object", properties: {} } }, - })) + // Post-process: Clean up messages and ensure tool call consistency + return cleanupMessages(messages) } function translateGeminiContentToOpenAI( @@ -402,124 +410,11 @@ function extractTextFromGeminiContent(content: GeminiContent): string { .join("\n\n") } -function translateGeminiToolsToOpenAI( - geminiTools?: Array, -): Array | undefined { - if (!geminiTools || geminiTools.length === 0) return undefined - - const tools: Array = [] - for (const tool of geminiTools) { - // Handle standard function declarations - if (tool.functionDeclarations) { - for (const func of tool.functionDeclarations) { - // Validate that function name exists and is not empty - if ( - !func.name - || typeof func.name !== "string" - || func.name.trim() === "" - ) { - continue - } - - // Ensure parameters is always a valid object - - const validParameters = func.parametersJsonSchema - || func.parameters || { type: "object", properties: {} } - - tools.push({ - type: "function", - function: { - name: func.name, - description: func.description, - parameters: validParameters, - }, - }) - } - } - - // Handle googleSearch tool (special case) - if (tool.googleSearch !== undefined) { - tools.push({ - type: "function", - function: { - name: "google_web_search", - description: - "Performs a web search using Google Search (via the Gemini API) and returns the results. This tool is useful for finding information on the internet based on a query.", - parameters: { - type: "object", - properties: { - query: { - type: "string", - description: "The search query to find information on the web.", - }, - }, - required: ["query"], - }, - }, - }) - } - - // Handle urlContext tool (special case for web_fetch) - // Note: GitHub Copilot API doesn't support web_fetch functionality - // Skip this tool to avoid "Failed to create chat completions" errors - if (tool.urlContext !== undefined) { - continue - } - } - - return tools.length > 0 ? tools : undefined -} - -function translateGeminiToolConfigToOpenAI( - toolConfig?: GeminiRequest["toolConfig"], -): ChatCompletionsPayload["tool_choice"] { - if (!toolConfig) return undefined - - const mode = toolConfig.functionCallingConfig.mode - switch (mode) { - case "AUTO": { - return "auto" - } - case "ANY": { - return "required" - } - case "NONE": { - return "none" - } - default: { - return undefined - } - } -} - // Response translation: OpenAI -> Gemini // Helper function to deduplicate tool responses - remove duplicate tool_call_ids // The problem was our logic was CREATING duplicates instead of preventing them -function ensureToolCallResponseMatch(messages: Array): Array { - const result: Array = [] - const seenToolCallIds = new Set() // Track processed tool_call_ids to avoid duplicates - - for (const message of messages) { - if (message.role === "tool" && message.tool_call_id) { - const toolCallId = message.tool_call_id - - // Only keep the FIRST response for each tool_call_id (deduplicate) - if (!seenToolCallIds.has(toolCallId)) { - seenToolCallIds.add(toolCallId) - result.push(message) - } - // Skip any duplicate responses for the same tool_call_id - } else { - // Keep all non-tool messages as-is - result.push(message) - } - } - - return result -} - export function translateOpenAIToGemini( response: ChatCompletionResponse, ): GeminiResponse { @@ -614,145 +509,6 @@ function translateOpenAIMessageToGeminiContent( // Utility functions -function generateToolCallId(functionName: string): string { - return `call_${functionName}_${Date.now()}_${Math.random().toString(36).slice(2, 11)}` -} - -// Global accumulator for streaming tool call arguments -const streamingToolCallAccumulator = new Map< - number, - { - name: string - arguments: string - id?: string - } ->() - -// Helper function to try parsing and creating a function call -function tryCreateFunctionCall( - name: string, - argumentsStr: string, -): GeminiPart | null { - try { - const args = JSON.parse(argumentsStr) as Record - return { - functionCall: { - name, - args, - }, - } - } catch { - return null - } -} - -// Helper function to handle tool call with function name -function handleToolCallWithName(toolCall: { - index: number - id?: string - function: { - name: string - arguments?: string - } -}): GeminiPart | null { - const accumulatedArgs = toolCall.function.arguments || "" - - streamingToolCallAccumulator.set(toolCall.index, { - name: toolCall.function.name, - arguments: accumulatedArgs, - id: toolCall.id, - }) - - // If we already have arguments, try to process immediately (for non-streaming models like Gemini) - if (accumulatedArgs) { - const functionCall = tryCreateFunctionCall( - toolCall.function.name, - accumulatedArgs, - ) - if (functionCall) { - // Clear the accumulator for this index since we've successfully processed it - streamingToolCallAccumulator.delete(toolCall.index) - return functionCall - } - } - - return null -} - -// Helper function to handle tool call argument accumulation -function handleToolCallAccumulation(toolCall: { - index: number - function?: { - arguments?: string - } -}): GeminiPart | null { - const existingAccumulated = streamingToolCallAccumulator.get(toolCall.index) - - if (existingAccumulated && toolCall.function?.arguments) { - existingAccumulated.arguments += toolCall.function.arguments - - const functionCall = tryCreateFunctionCall( - existingAccumulated.name, - existingAccumulated.arguments, - ) - if (functionCall) { - // Clear the accumulator for this index since we've successfully processed it - streamingToolCallAccumulator.delete(toolCall.index) - return functionCall - } - } - - return null -} - -// Helper function to process tool calls in streaming chunks with argument accumulation -function processToolCalls( - toolCalls: Array<{ - index: number - id?: string - type?: "function" - function?: { - name?: string - arguments?: string - } - }>, -): Array { - const parts: Array = [] - - for (const toolCall of toolCalls) { - // Debug: Log streaming tool call arguments to verify what GitHub Copilot returns - if (process.env.DEBUG_GEMINI_REQUESTS === "true") { - console.log( - `[DEBUG STREAM] Tool call - name: ${toolCall.function?.name}, arguments: "${toolCall.function?.arguments}", type: ${typeof toolCall.function?.arguments}, truthy: ${Boolean(toolCall.function?.arguments)}`, - ) - } - - // If this chunk has a function name, it's the start of a new tool call - if (toolCall.function?.name && toolCall.function.name.trim() !== "") { - const functionCall = handleToolCallWithName({ - index: toolCall.index, - id: toolCall.id, - function: { - name: toolCall.function.name, - arguments: toolCall.function.arguments, - }, - }) - if (functionCall) { - parts.push(functionCall) - } - continue - } - - // If we have existing accumulated data and this chunk has arguments, append them - const functionCall = handleToolCallAccumulation(toolCall) - if (functionCall) { - parts.push(functionCall) - } - } - - return parts -} - // Helper function to create usage metadata function createUsageMetadata(chunk: ChatCompletionChunk): GeminiUsageMetadata { return { @@ -763,20 +519,23 @@ function createUsageMetadata(chunk: ChatCompletionChunk): GeminiUsageMetadata { } // Helper function to process chunk parts -function processChunkParts(choice: { - delta: { - content?: string | null - tool_calls?: Array<{ - index: number - id?: string - type?: "function" - function?: { - name?: string - arguments?: string - } - }> - } -}): Array { +function processChunkParts( + choice: { + delta: { + content?: string | null + tool_calls?: Array<{ + index: number + id?: string + type?: "function" + function?: { + name?: string + arguments?: string + } + }> + } + }, + accumulator: ToolCallAccumulator, +): Array { const parts: Array = [] if (choice.delta.content) { @@ -784,7 +543,9 @@ function processChunkParts(choice: { } if (choice.delta.tool_calls) { - parts.push(...processToolCalls(choice.delta.tool_calls)) + parts.push( + ...processToolCallsWithAccumulator(choice.delta.tool_calls, accumulator), + ) } return parts @@ -819,22 +580,25 @@ function createGeminiCandidate( } // Helper function to handle parts processing and validation -function processParts(choice: { - finish_reason: "stop" | "length" | "tool_calls" | "content_filter" | null - delta: { - content?: string | null - tool_calls?: Array<{ - index: number - id?: string - type?: "function" - function?: { - name?: string - arguments?: string - } - }> - } -}): Array | null { - const parts = processChunkParts(choice) +function processParts( + choice: { + finish_reason: "stop" | "length" | "tool_calls" | "content_filter" | null + delta: { + content?: string | null + tool_calls?: Array<{ + index: number + id?: string + type?: "function" + function?: { + name?: string + arguments?: string + } + }> + } + }, + accumulator: ToolCallAccumulator, +): Array | null { + const parts = processChunkParts(choice, accumulator) if (parts.length === 0 && !choice.finish_reason) { return null @@ -873,7 +637,10 @@ function buildGeminiResponse( } // Stream translation: OpenAI Chunk -> Gemini Stream Response -export function translateOpenAIChunkToGemini(chunk: ChatCompletionChunk): { +export function translateOpenAIChunkToGemini( + chunk: ChatCompletionChunk, + accumulator: ToolCallAccumulator, +): { candidates: Array usageMetadata?: GeminiUsageMetadata } | null { @@ -883,7 +650,7 @@ export function translateOpenAIChunkToGemini(chunk: ChatCompletionChunk): { const choice = chunk.choices[0] - const parts = processParts(choice) + const parts = processParts(choice, accumulator) if (!parts) { return null } From de096e85614925a98641c0f05bf3912b84c60338 Mon Sep 17 00:00:00 2001 From: cpf Date: Wed, 1 Oct 2025 21:27:26 +0800 Subject: [PATCH 10/16] refactor: remove unused tool response deduplication function and clean up imports --- src/lib/tool-call-utils.ts | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/src/lib/tool-call-utils.ts b/src/lib/tool-call-utils.ts index f642b5e9d..01cb34a55 100644 --- a/src/lib/tool-call-utils.ts +++ b/src/lib/tool-call-utils.ts @@ -4,7 +4,7 @@ import type { GeminiContent, GeminiPart, } from "~/routes/generate-content/types" -import type { Tool, Message } from "~/services/copilot/create-chat-completions" +import type { Tool } from "~/services/copilot/create-chat-completions" // Tool declaration generation - moved from translation.ts export function translateGeminiToolsToOpenAI( @@ -97,32 +97,6 @@ export function translateGeminiToolConfigToOpenAI( } } -// Tool response deduplication - moved from translation.ts -export function ensureToolCallResponseMatch( - messages: Array, -): Array { - const result: Array = [] - const seenToolCallIds = new Set() // Track processed tool_call_ids to avoid duplicates - - for (const message of messages) { - if (message.role === "tool" && message.tool_call_id) { - const toolCallId = message.tool_call_id - - // Only keep the FIRST response for each tool_call_id (deduplicate) - if (!seenToolCallIds.has(toolCallId)) { - seenToolCallIds.add(toolCallId) - result.push(message) - } - // Skip any duplicate responses for the same tool_call_id - } else { - // Keep all non-tool messages as-is - result.push(message) - } - } - - return result -} - // Utility function to generate unique tool call IDs - moved from translation.ts // Generate IDs within 40 character limit (API constraint) export function generateToolCallId(_functionName: string): string { @@ -132,6 +106,8 @@ export function generateToolCallId(_functionName: string): string { } // Helper function to try parsing and creating a function call - moved from translation.ts +// NOTE: Used internally by ToolCallAccumulator.handleToolCallWithName() and handleToolCallAccumulation() +// knip may report this as unused, but it's called within this module's class methods export function tryCreateFunctionCall( name: string, argumentsStr: string, From a54b7a6856065e46ab464f4eaad0fdbca819fba7 Mon Sep 17 00:00:00 2001 From: cpf Date: Wed, 1 Oct 2025 21:40:47 +0800 Subject: [PATCH 11/16] refactor: update comments in ToolCallAccumulator for clarity and consistency --- src/lib/tool-call-utils.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/lib/tool-call-utils.ts b/src/lib/tool-call-utils.ts index 01cb34a55..c17117cf9 100644 --- a/src/lib/tool-call-utils.ts +++ b/src/lib/tool-call-utils.ts @@ -151,8 +151,7 @@ export function synthesizeToolsFromContents( } /** - * 工具调用状态管理器 - * 用于流式响应中工具调用参数的增量累积 + * Tool call state manager for incremental parameter accumulation in streaming responses */ export class ToolCallAccumulator { private accumulator = new Map< @@ -165,7 +164,7 @@ export class ToolCallAccumulator { >() /** - * 处理带有函数名的工具调用（新工具调用的开始） + * Handle tool call with function name (start of new tool call) */ handleToolCallWithName(toolCall: { index: number @@ -200,7 +199,7 @@ export class ToolCallAccumulator { } /** - * 处理工具调用参数累积（追加参数片段） + * Handle tool call parameter accumulation (append argument fragments) */ handleToolCallAccumulation(toolCall: { index: number @@ -228,7 +227,7 @@ export class ToolCallAccumulator { } /** - * 清理所有累积状态（用于流结束或错误重置） + * Clear all accumulated state (for stream end or error reset) */ clear(): void { this.accumulator.clear() @@ -236,8 +235,8 @@ export class ToolCallAccumulator { } /** - * 处理工具调用数组，生成 Gemini 格式的部分 - * 支持完整参数和分片参数两种模式 + * Process tool calls array and generate Gemini format parts + * Supports both complete parameters and fragmented parameters modes */ export function processToolCalls( toolCalls: Array<{ From fcd21540c5d1debc22e6947aeb7ef50d4a34341c Mon Sep 17 00:00:00 2001 From: cpf Date: Wed, 1 Oct 2025 21:54:35 +0800 Subject: [PATCH 12/16] refactor: improve comments for clarity in test files --- tests/generate-content/test-types.ts | 1 + .../translation-coverage.test.ts | 313 ++++++++++++++++-- tests/generate-content/translation.test.ts | 4 +- 3 files changed, 290 insertions(+), 28 deletions(-) diff --git a/tests/generate-content/test-types.ts b/tests/generate-content/test-types.ts index 94dddcd08..4fc8df3f7 100644 --- a/tests/generate-content/test-types.ts +++ b/tests/generate-content/test-types.ts @@ -35,6 +35,7 @@ export interface CapturedPayload extends Record { type: string function: { name: string; arguments: string } }> + tool_call_id?: string }> tools?: Array<{ type: string diff --git a/tests/generate-content/translation-coverage.test.ts b/tests/generate-content/translation-coverage.test.ts index 00eb6239a..b36cdb2ce 100644 --- a/tests/generate-content/translation-coverage.test.ts +++ b/tests/generate-content/translation-coverage.test.ts @@ -27,7 +27,7 @@ test("processes function response arrays with tool call matching", async () => { }, })) - // Test nested function response arrays (lines 105-134) + // Should correctly process nested function response arrays const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { contents: [ { role: "user", parts: [{ text: "Call function" }] }, @@ -56,8 +56,15 @@ test("processes function response arrays with tool call matching", async () => { }) expect(res.status).toBe(200) - // This test validates that the nested array structure is processed correctly - expect(capturedPayload.messages?.length).toBeGreaterThan(0) + // Verify nested array structure is processed correctly + const messages = capturedPayload.messages ?? [] + expect(messages.length).toBeGreaterThan(0) + + // Should successfully parse and process nested function response arrays + // The actual message structure depends on cleanup logic + // Key is that the request succeeds and messages are generated + const userMessages = messages.filter((m) => m.role === "user") + expect(userMessages.length).toBeGreaterThan(0) }) test("handles function response without matching tool call", async () => { @@ -79,7 +86,7 @@ test("handles function response without matching tool call", async () => { }, })) - // Test case where function response has no matching tool call (line 170) + // Should skip function responses without matching tool calls const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { contents: [ { role: "user", parts: [{ text: "Call function" }] }, @@ -100,7 +107,13 @@ test("handles function response without matching tool call", async () => { expect(res.status).toBe(200) const toolMessages = capturedPayload.messages?.filter((m) => m.role === "tool") ?? [] - expect(toolMessages.length).toBe(0) // No matching tool call, so no tool message + expect(toolMessages.length).toBe(0) + + // Verify user messages are still processed + const userMessages = + capturedPayload.messages?.filter((m) => m.role === "user") ?? [] + expect(userMessages.length).toBeGreaterThan(0) + expect(userMessages[0]?.content).toContain("Call function") }) test("handles empty content merging fallback", async () => { @@ -122,7 +135,7 @@ test("handles empty content merging fallback", async () => { }, })) - // Test empty content fallback (lines 248-249) + // Should merge empty and whitespace-only content correctly const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { contents: [ { role: "user", parts: [{ text: "" }] }, // Empty text @@ -134,8 +147,10 @@ test("handles empty content merging fallback", async () => { expect(res.status).toBe(200) const userMessages = capturedPayload.messages?.filter((m) => m.role === "user") ?? [] - expect(userMessages.length).toBe(1) // Should merge into one message + expect(userMessages.length).toBe(1) expect(userMessages[0]?.content).toContain("actual question") + // Ensure empty/whitespace content doesn't appear in merged message + expect(userMessages[0]?.content).not.toMatch(/^\s*$/) }) test("handles complex content that cannot be merged", async () => { @@ -157,7 +172,7 @@ test("handles complex content that cannot be merged", async () => { }, })) - // Test complex content merging fallback (line 238) + // Should handle complex content mixing text and function responses const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { contents: [ { role: "user", parts: [{ text: "First message" }] }, @@ -177,8 +192,15 @@ test("handles complex content that cannot be merged", async () => { }) expect(res.status).toBe(200) - // This test validates the content processing logic handles complex scenarios - expect(capturedPayload.messages?.length).toBeGreaterThan(0) + const messages = capturedPayload.messages ?? [] + expect(messages.length).toBeGreaterThan(0) + + // Verify text messages are merged but function responses are handled separately + const userMessages = messages.filter((m) => m.role === "user") + expect(userMessages.length).toBeGreaterThan(0) + const mergedContent = userMessages.map((m) => m.content).join(" ") + expect(mergedContent).toContain("First message") + expect(mergedContent).toContain("Second message") }) test("maps unsupported Gemini model names to supported ones", async () => { @@ -200,7 +222,7 @@ test("maps unsupported Gemini model names to supported ones", async () => { }, })) - // Test model mapping (lines 29-37) + // Should map unsupported model names to supported equivalents const res = await makeRequest( "/v1beta/models/gemini-2.5-flash:generateContent", { @@ -209,7 +231,7 @@ test("maps unsupported Gemini model names to supported ones", async () => { ) expect(res.status).toBe(200) - expect(capturedPayload.model).toBe("gemini-2.0-flash-001") // Should be mapped + expect(capturedPayload.model).toBe("gemini-2.0-flash-001") }) test("preserves supported model names without mapping", async () => { @@ -231,7 +253,7 @@ test("preserves supported model names without mapping", async () => { }, })) - // Test model mapping preservation (line 36) + // Should preserve already supported model names const res = await makeRequest( "/v1beta/models/gemini-1.5-pro:generateContent", { @@ -240,7 +262,7 @@ test("preserves supported model names without mapping", async () => { ) expect(res.status).toBe(200) - expect(capturedPayload.model).toBe("gemini-1.5-pro") // Should remain unchanged + expect(capturedPayload.model).toBe("gemini-1.5-pro") }) test("handles tool call cleanup with incomplete tool calls", async () => { @@ -262,7 +284,7 @@ test("handles tool call cleanup with incomplete tool calls", async () => { }, })) - // Test incomplete tool call cleanup (lines 295-296) + // Should clean up incomplete tool calls (tool_calls without responses) const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { contents: [ { role: "user", parts: [{ text: "Search something" }] }, @@ -270,16 +292,20 @@ test("handles tool call cleanup with incomplete tool calls", async () => { role: "model", parts: [{ functionCall: { name: "search", args: { query: "test" } } }], }, - // No function response - incomplete tool call that should be cleaned up { role: "user", parts: [{ text: "What did you find?" }] }, ], }) expect(res.status).toBe(200) - // The incomplete tool call should be cleaned up + // Incomplete tool calls should be removed const assistantMessages = capturedPayload.messages?.filter((m) => m.role === "assistant") ?? [] - expect(assistantMessages.length).toBe(0) // Should be cleaned up + expect(assistantMessages.length).toBe(0) + + // User messages should still be present + const userMessages = + capturedPayload.messages?.filter((m) => m.role === "user") ?? [] + expect(userMessages.length).toBeGreaterThan(0) }) test("processes inline data with inlineData field", async () => { @@ -301,7 +327,7 @@ test("processes inline data with inlineData field", async () => { }, })) - // Test inline data processing (lines 374, 377-381) + // Should process inline data (base64-encoded images) correctly const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { contents: [ { @@ -320,8 +346,14 @@ test("processes inline data with inlineData field", async () => { }) expect(res.status).toBe(200) - // This test validates inline data processing - expect(capturedPayload.messages?.length).toBeGreaterThan(0) + expect(capturedPayload.messages?.length).toBe(1) + + const userMessage = capturedPayload.messages?.[0] + expect(userMessage?.role).toBe("user") + // Content should include both text and image data + const content = userMessage?.content + expect(content).toBeDefined() + expect(typeof content === "string" || Array.isArray(content)).toBe(true) }) test("handles streaming tool calls with incomplete arguments", async () => { @@ -366,7 +398,7 @@ test("accumulates streaming tool call arguments correctly", async () => { }, })) - // Test streaming arguments accumulation (lines 566-579) + // Should handle streaming arguments accumulation correctly const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { contents: [{ role: "user", parts: [{ text: "Search for something" }] }], }) @@ -394,7 +426,7 @@ test("handles Google Search tool processing", async () => { }, })) - // Test Google Search tool handling (lines 442-459) + // Should handle Google Search tool configuration and processing const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { tools: [ { @@ -410,12 +442,20 @@ test("handles Google Search tool processing", async () => { }) expect(res.status).toBe(200) - // This test validates Google Search tool processing logic - expect(capturedPayload.messages?.length).toBeGreaterThan(0) + expect(capturedPayload.messages?.length).toBe(1) + + const userMessage = capturedPayload.messages?.[0] + expect(userMessage?.role).toBe("user") + expect(userMessage?.content).toContain("latest news") + + // Google Search tool is Gemini-specific and gets translated + // It may or may not appear in the tools array depending on translation logic + // The key is that the request succeeds + expect(capturedPayload.messages).toBeDefined() }) test("handles translation errors gracefully", async () => { - // Mock a scenario that would trigger error handling (lines 702-703, 881, 904) + // Should return appropriate error status when Copilot API fails await mock.module("~/services/copilot/create-chat-completions", () => ({ createChatCompletions: () => { throw new Error("Copilot API error") @@ -468,3 +508,224 @@ test("handles malformed tool calls in content processing", async () => { expect(res.status).toBe(200) // Should handle malformed calls gracefully }) + +// Real scenario tests for multi-turn tool calls and deduplication + +test("handles multi-turn tool call conversation correctly", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "Result processed" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 50, completion_tokens: 10, total_tokens: 60 }, + } + }, + })) + + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "Read file A" }] }, + { + role: "model", + parts: [ + { functionCall: { name: "readFile", args: { path: "a.txt" } } }, + ], + }, + { + role: "user", + parts: [ + { + functionResponse: { + name: "readFile", + response: { content: "Content of A" }, + }, + }, + ], + }, + { + role: "model", + parts: [{ text: "File A contains: Content of A" }], + }, + { role: "user", parts: [{ text: "Now read file B" }] }, + { + role: "model", + parts: [ + { functionCall: { name: "readFile", args: { path: "b.txt" } } }, + ], + }, + { + role: "user", + parts: [ + { + functionResponse: { + name: "readFile", + response: { content: "Content of B" }, + }, + }, + ], + }, + ], + }) + + expect(res.status).toBe(200) + + // Verify message structure: user, assistant+tool_call, tool, assistant, user, assistant+tool_call, tool + const messages = capturedPayload.messages ?? [] + expect(messages.length).toBeGreaterThanOrEqual(5) + + // Verify tool call ID consistency + const assistantWithTools = messages.filter( + (m) => m.role === "assistant" && m.tool_calls, + ) + expect(assistantWithTools.length).toBeGreaterThanOrEqual(2) + + const toolMessages = messages.filter((m) => m.role === "tool") + expect(toolMessages.length).toBeGreaterThanOrEqual(2) + + // Each tool message should reference a tool_call_id + for (const toolMsg of toolMessages) { + expect(toolMsg.tool_call_id).toBeDefined() + expect(typeof toolMsg.tool_call_id).toBe("string") + } +}) + +test("handles duplicate tool responses by deduplication", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "Processed" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }, + } + }, + })) + + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "Call function" }] }, + { + role: "model", + parts: [ + { functionCall: { name: "testFunc", args: { param: "value1" } } }, + { functionCall: { name: "testFunc2", args: { param: "value2" } } }, + ], + }, + { + role: "user", + parts: [ + { + functionResponse: { + name: "testFunc", + response: { result: "first" }, + }, + }, + { + functionResponse: { + name: "testFunc2", + response: { result: "second" }, + }, + }, + // Duplicate response - should be deduplicated + { + functionResponse: { + name: "testFunc", + response: { result: "duplicate" }, + }, + }, + ], + }, + ], + }) + + expect(res.status).toBe(200) + + // Verify deduplication: should have exactly 2 tool messages (not 3) + const messages = capturedPayload.messages ?? [] + const toolMessages = messages.filter((m) => m.role === "tool") + + // Count unique tool_call_ids + const toolCallIds = new Set( + toolMessages.map((m) => m.tool_call_id).filter(Boolean), + ) + expect(toolCallIds.size).toBeLessThanOrEqual(2) +}) + +test("verifies tool_call_id length constraint (≤40 characters)", async () => { + let capturedPayload: CapturedPayload = {} as CapturedPayload + await mock.module("~/services/copilot/create-chat-completions", () => ({ + createChatCompletions: (payload: CapturedPayload) => { + capturedPayload = payload + return { + id: "x", + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + } + }, + })) + + const res = await makeRequest("/v1beta/models/gemini-pro:generateContent", { + contents: [ + { role: "user", parts: [{ text: "Call a function" }] }, + { + role: "model", + parts: [ + { + functionCall: { + name: "veryLongFunctionNameThatMightCauseIssues", + args: { param: "test" }, + }, + }, + ], + }, + { + role: "user", + parts: [ + { + functionResponse: { + name: "veryLongFunctionNameThatMightCauseIssues", + response: { result: "ok" }, + }, + }, + ], + }, + ], + }) + + expect(res.status).toBe(200) + + const messages = capturedPayload.messages ?? [] + const assistantWithTools = messages.filter( + (m) => m.role === "assistant" && m.tool_calls, + ) + + // Verify all generated tool_call_ids are within limit + for (const msg of assistantWithTools) { + if (msg.tool_calls) { + for (const toolCall of msg.tool_calls) { + expect(toolCall.id.length).toBeLessThanOrEqual(40) + } + } + } +}) diff --git a/tests/generate-content/translation.test.ts b/tests/generate-content/translation.test.ts index 559e25146..85608679c 100644 --- a/tests/generate-content/translation.test.ts +++ b/tests/generate-content/translation.test.ts @@ -27,7 +27,7 @@ test("processes toolConfig AUTO/ANY/NONE mapping end-to-end", async () => { }, })) - // toolConfig 需要有 tools 才会被处理，所以添加 tools + // toolConfig requires tools to be processed, so add tools to request const baseRequest = { tools: [ { @@ -264,7 +264,7 @@ test("handles empty contents gracefully", async () => { contents: [], }) - // 空 contents 会导致翻译过程中出错，返回 500 + // Empty contents cause translation error, expect 500 status expect(res.status).toBe(500) }) From 4abb0a5ce86f434597071b9b9f375764c191fbc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E9=B9=8F=E9=A3=9E=20Cui=20Pengfei?= <1311541+cuipengfei@users.noreply.github.com> Date: Wed, 1 Oct 2025 22:12:09 +0800 Subject: [PATCH 13/16] Update tests/generate-content/validation-and-routing.test.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tests/generate-content/validation-and-routing.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generate-content/validation-and-routing.test.ts b/tests/generate-content/validation-and-routing.test.ts index d391e60f4..de11b817e 100644 --- a/tests/generate-content/validation-and-routing.test.ts +++ b/tests/generate-content/validation-and-routing.test.ts @@ -222,7 +222,7 @@ test("handles malformed JSON in request body", async () => { }, ) - // JSON 解析错误会返回 500 + // JSON parsing errors will return 500 expect(res.status).toBe(500) }) From 7b3db7a7e20bd9e6e4f56c7722cc2f381ed62dfe Mon Sep 17 00:00:00 2001 From: cpf Date: Wed, 1 Oct 2025 22:18:21 +0800 Subject: [PATCH 14/16] fix: ensure newline at end of file in server.ts --- src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.ts b/src/server.ts index 479fc9e20..b7fea00e4 100644 --- a/src/server.ts +++ b/src/server.ts @@ -35,4 +35,4 @@ server.route("/v1/responses", responsesRoutes) server.route("/v1/messages", messageRoutes) // Gemini -server.route("/", geminiRouter) \ No newline at end of file +server.route("/", geminiRouter) From 48a20c152f8662733ca71922a3b4d5341f6154db Mon Sep 17 00:00:00 2001 From: cpf Date: Wed, 1 Oct 2025 22:26:38 +0800 Subject: [PATCH 15/16] fix: handle missing model in token counting endpoint --- src/routes/generate-content/handler.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/routes/generate-content/handler.ts b/src/routes/generate-content/handler.ts index 88ec39473..26c3c3a43 100644 --- a/src/routes/generate-content/handler.ts +++ b/src/routes/generate-content/handler.ts @@ -300,7 +300,16 @@ export async function handleGeminiCountTokens(c: Context) { const openAIPayload = translateGeminiCountTokensToOpenAI(geminiPayload, model) - const tokenCounts = getTokenCount(openAIPayload.messages) + // Find the full Model object from state + const selectedModel = state.models?.data.find((m) => m.id === model) + + if (!selectedModel) { + // Fallback: return minimal token count if model not found + const geminiResponse = translateTokenCountToGemini(10) + return c.json(geminiResponse) + } + + const tokenCounts = await getTokenCount(openAIPayload, selectedModel) const totalTokens = tokenCounts.input + tokenCounts.output const geminiResponse = translateTokenCountToGemini(totalTokens) From 3284346f2f8841f607e3499f37eab166b50dc510 Mon Sep 17 00:00:00 2001 From: cpf Date: Thu, 2 Oct 2025 11:13:34 +0800 Subject: [PATCH 16/16] fix tests --- tests/generate-content/core-functionality.test.ts | 15 ++++++++++++++- tests/generate-content/route-routing.test.ts | 15 ++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/tests/generate-content/core-functionality.test.ts b/tests/generate-content/core-functionality.test.ts index 7f05608e8..a88cce5b4 100644 --- a/tests/generate-content/core-functionality.test.ts +++ b/tests/generate-content/core-functionality.test.ts @@ -27,7 +27,20 @@ test("translates request and uses local tokenizer without downstream call", asyn }, })) await mock.module("~/lib/tokenizer", () => ({ - getTokenCount: (_: unknown) => ({ input: 2, output: 3 }), + getTokenCount: async (_p: unknown, _m: unknown) => + Promise.resolve({ input: 2, output: 3 }), + })) + await mock.module("~/lib/state", () => ({ + state: { + models: { + data: [ + { + id: "gemini-pro", + capabilities: { tokenizer: "o200k_base" }, + }, + ], + }, + }, })) const { server } = (await import("~/server")) as { server: TestServer } diff --git a/tests/generate-content/route-routing.test.ts b/tests/generate-content/route-routing.test.ts index cb73173d8..95c2d673c 100644 --- a/tests/generate-content/route-routing.test.ts +++ b/tests/generate-content/route-routing.test.ts @@ -66,7 +66,20 @@ test("routes to stream endpoint based on URL keyword", async () => { test("routes to countTokens endpoint based on URL keyword", async () => { await mock.module("~/lib/tokenizer", () => ({ - getTokenCount: (_: unknown) => ({ input: 2, output: 3 }), + getTokenCount: async (_p: unknown, _m: unknown) => + Promise.resolve({ input: 2, output: 3 }), + })) + await mock.module("~/lib/state", () => ({ + state: { + models: { + data: [ + { + id: "gemini-pro", + capabilities: { tokenizer: "o200k_base" }, + }, + ], + }, + }, })) await mock.module("~/lib/rate-limit", () => ({ checkRateLimit: () => {},