|
| 1 | +import * as assert from "assert" |
| 2 | + |
| 3 | +import { RooCodeEventName, type ClineMessage } from "@roo-code/types" |
| 4 | + |
| 5 | +import { setDefaultSuiteTimeout } from "../test-utils" |
| 6 | +import { waitUntilCompleted } from "../utils" |
| 7 | + |
| 8 | +const GEMINI_API_KEY = process.env.GEMINI_API_KEY ?? process.env.GOOGLE_API_KEY |
| 9 | +const GEMINI_MODEL_ID = process.env.GEMINI_MODEL_ID ?? "gemini-3-flash-preview" |
| 10 | + |
| 11 | +type FunctionDeclaration = { |
| 12 | + name: string |
| 13 | + parametersJsonSchema?: Record<string, unknown> |
| 14 | +} |
| 15 | + |
| 16 | +type GeminiToolConfig = { |
| 17 | + functionCallingConfig?: { |
| 18 | + mode?: string |
| 19 | + allowedFunctionNames?: string[] |
| 20 | + } |
| 21 | +} |
| 22 | + |
| 23 | +type CapturedGeminiRequest = { |
| 24 | + model?: string |
| 25 | + lastUserMessage: string |
| 26 | + thinkingConfig?: Record<string, unknown> |
| 27 | + toolConfig?: GeminiToolConfig |
| 28 | + hasTools: boolean |
| 29 | + toolDeclarationCount: number |
| 30 | + functionDeclarations: FunctionDeclaration[] |
| 31 | +} |
| 32 | + |
| 33 | +function findInvalidSchemaPatterns(schema: unknown, path = ""): string[] { |
| 34 | + if (!schema || typeof schema !== "object" || Array.isArray(schema)) { |
| 35 | + return [] |
| 36 | + } |
| 37 | + |
| 38 | + const obj = schema as Record<string, unknown> |
| 39 | + const violations: string[] = [] |
| 40 | + |
| 41 | + if ("additionalProperties" in obj) { |
| 42 | + violations.push(`${path}.additionalProperties (stripped for Gemini compatibility)`) |
| 43 | + } |
| 44 | + |
| 45 | + if ("default" in obj) { |
| 46 | + violations.push(`${path}.default (stripped for Gemini compatibility)`) |
| 47 | + } |
| 48 | + |
| 49 | + if ("$schema" in obj) { |
| 50 | + violations.push(`${path}.$schema (JSON Schema metadata stripped for Gemini compatibility)`) |
| 51 | + } |
| 52 | + |
| 53 | + if ("type" in obj && Array.isArray(obj.type)) { |
| 54 | + violations.push(`${path}.type is an array ${JSON.stringify(obj.type)} (Gemini requires a single string type)`) |
| 55 | + } |
| 56 | + |
| 57 | + for (const [key, value] of Object.entries(obj)) { |
| 58 | + if (key === "properties" && value && typeof value === "object") { |
| 59 | + for (const [propName, propSchema] of Object.entries(value as Record<string, unknown>)) { |
| 60 | + violations.push(...findInvalidSchemaPatterns(propSchema, `${path}.properties.${propName}`)) |
| 61 | + } |
| 62 | + } else if (key === "items") { |
| 63 | + violations.push(...findInvalidSchemaPatterns(value, `${path}.items`)) |
| 64 | + } else if (key === "anyOf" || key === "oneOf" || key === "allOf") { |
| 65 | + violations.push(`${path}.${key} (collapsed for Gemini compatibility)`) |
| 66 | + if (Array.isArray(value)) { |
| 67 | + value.forEach((item, i) => violations.push(...findInvalidSchemaPatterns(item, `${path}.${key}[${i}]`))) |
| 68 | + } |
| 69 | + } |
| 70 | + } |
| 71 | + |
| 72 | + return violations |
| 73 | +} |
| 74 | + |
| 75 | +function getRequestUrl(input: RequestInfo | URL): string { |
| 76 | + return typeof input === "string" ? input : input instanceof URL ? input.href : (input as Request).url |
| 77 | +} |
| 78 | + |
| 79 | +function isUrlWithOrigin(rawUrl: string, expectedOrigin: string): boolean { |
| 80 | + try { |
| 81 | + return new URL(rawUrl).origin === expectedOrigin |
| 82 | + } catch { |
| 83 | + return false |
| 84 | + } |
| 85 | +} |
| 86 | + |
| 87 | +function isGeminiGenerateContentUrl(rawUrl: string): boolean { |
| 88 | + try { |
| 89 | + const pathname = new URL(rawUrl).pathname |
| 90 | + return pathname.includes(":streamGenerateContent") || pathname.includes(":generateContent") |
| 91 | + } catch { |
| 92 | + return false |
| 93 | + } |
| 94 | +} |
| 95 | + |
| 96 | +function extractGeminiModel(rawUrl: string): string | undefined { |
| 97 | + try { |
| 98 | + const pathname = new URL(rawUrl).pathname |
| 99 | + const match = pathname.match(/\/models\/([^:]+):(streamGenerateContent|generateContent)$/) |
| 100 | + return match?.[1] |
| 101 | + } catch { |
| 102 | + return undefined |
| 103 | + } |
| 104 | +} |
| 105 | + |
| 106 | +function extractLastUserMessage( |
| 107 | + contents?: Array<{ |
| 108 | + role?: string |
| 109 | + parts?: Array<{ text?: string }> |
| 110 | + }>, |
| 111 | +): string { |
| 112 | + const lastUser = [...(contents ?? [])].reverse().find((content) => content.role === "user") |
| 113 | + |
| 114 | + if (!lastUser?.parts) { |
| 115 | + return "" |
| 116 | + } |
| 117 | + |
| 118 | + return lastUser.parts |
| 119 | + .map((part) => (typeof part?.text === "string" ? part.text : JSON.stringify(part ?? ""))) |
| 120 | + .join("") |
| 121 | +} |
| 122 | + |
| 123 | +function installGeminiRequestCapture(capture: CapturedGeminiRequest[], baseUrl: string): () => void { |
| 124 | + const originalFetch = globalThis.fetch |
| 125 | + const targetOrigin = new URL(baseUrl).origin |
| 126 | + |
| 127 | + globalThis.fetch = async function (input: RequestInfo | URL, init?: RequestInit): Promise<Response> { |
| 128 | + const url = getRequestUrl(input) |
| 129 | + |
| 130 | + if (isUrlWithOrigin(url, targetOrigin) && isGeminiGenerateContentUrl(url)) { |
| 131 | + const body = init?.body && typeof init.body === "string" ? JSON.parse(init.body) : {} |
| 132 | + const tools = Array.isArray(body.tools) ? body.tools : [] |
| 133 | + const functionDeclarations: FunctionDeclaration[] = tools.flatMap( |
| 134 | + (tool: { functionDeclarations?: FunctionDeclaration[] }) => |
| 135 | + Array.isArray(tool.functionDeclarations) ? tool.functionDeclarations : [], |
| 136 | + ) |
| 137 | + |
| 138 | + capture.push({ |
| 139 | + model: extractGeminiModel(url), |
| 140 | + lastUserMessage: extractLastUserMessage(body.contents), |
| 141 | + thinkingConfig: |
| 142 | + body.generationConfig && typeof body.generationConfig === "object" |
| 143 | + ? (body.generationConfig.thinkingConfig as Record<string, unknown> | undefined) |
| 144 | + : undefined, |
| 145 | + toolConfig: |
| 146 | + body.toolConfig && typeof body.toolConfig === "object" |
| 147 | + ? (body.toolConfig as GeminiToolConfig) |
| 148 | + : undefined, |
| 149 | + hasTools: tools.length > 0, |
| 150 | + toolDeclarationCount: functionDeclarations.length, |
| 151 | + functionDeclarations, |
| 152 | + }) |
| 153 | + } |
| 154 | + |
| 155 | + return originalFetch.call(globalThis, input, init as RequestInit) |
| 156 | + } as typeof globalThis.fetch |
| 157 | + |
| 158 | + return () => { |
| 159 | + globalThis.fetch = originalFetch |
| 160 | + } |
| 161 | +} |
| 162 | + |
| 163 | +suite("Gemini provider", function () { |
| 164 | + setDefaultSuiteTimeout(this) |
| 165 | + |
| 166 | + let restoreFetch: (() => void) | undefined |
| 167 | + const requests: CapturedGeminiRequest[] = [] |
| 168 | + |
| 169 | + setup(function () { |
| 170 | + const aimockUrl = process.env.AIMOCK_URL |
| 171 | + const isReplay = aimockUrl && process.env.AIMOCK_RECORD !== "true" |
| 172 | + const isRecordRun = aimockUrl && process.env.AIMOCK_RECORD === "true" && !!GEMINI_API_KEY |
| 173 | + // Live runs without aimock are not supported — GEMINI_MODEL_ID must match the fixture. |
| 174 | + if (!isReplay && !isRecordRun) { |
| 175 | + this.skip() |
| 176 | + } |
| 177 | + }) |
| 178 | + |
| 179 | + suiteSetup(() => { |
| 180 | + restoreFetch = installGeminiRequestCapture( |
| 181 | + requests, |
| 182 | + process.env.AIMOCK_URL || "https://generativelanguage.googleapis.com", |
| 183 | + ) |
| 184 | + }) |
| 185 | + |
| 186 | + suiteTeardown(async () => { |
| 187 | + restoreFetch?.() |
| 188 | + restoreFetch = undefined |
| 189 | + |
| 190 | + const aimockUrl = process.env.AIMOCK_URL |
| 191 | + const isRecord = process.env.AIMOCK_RECORD === "true" |
| 192 | + await globalThis.api.setConfiguration({ |
| 193 | + apiProvider: "openrouter" as const, |
| 194 | + openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!, |
| 195 | + openRouterModelId: "openai/gpt-4.1", |
| 196 | + ...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }), |
| 197 | + }) |
| 198 | + }) |
| 199 | + |
| 200 | + for (const reasoningEffort of ["high", "low", "disable"] as const) { |
| 201 | + test(`Should complete a task end-to-end using ${GEMINI_MODEL_ID} via Gemini provider with reasoning effort "${reasoningEffort}"`, async () => { |
| 202 | + requests.length = 0 |
| 203 | + |
| 204 | + const api = globalThis.api |
| 205 | + const aimockUrl = process.env.AIMOCK_URL |
| 206 | + const isRecord = process.env.AIMOCK_RECORD === "true" |
| 207 | + const promptTag = `gemini-e2e:reasoning-${reasoningEffort}` |
| 208 | + |
| 209 | + await api.setConfiguration({ |
| 210 | + apiProvider: "gemini" as const, |
| 211 | + geminiApiKey: aimockUrl && !isRecord ? "mock-key" : GEMINI_API_KEY!, |
| 212 | + apiModelId: GEMINI_MODEL_ID, |
| 213 | + enableReasoningEffort: reasoningEffort !== "disable", |
| 214 | + reasoningEffort: reasoningEffort, |
| 215 | + ...(aimockUrl && { googleGeminiBaseUrl: aimockUrl }), |
| 216 | + }) |
| 217 | + |
| 218 | + const messages: ClineMessage[] = [] |
| 219 | + const messageHandler = ({ message }: { message: ClineMessage }) => { |
| 220 | + if (message.type === "say" && message.partial === false) { |
| 221 | + messages.push(message) |
| 222 | + } |
| 223 | + } |
| 224 | + |
| 225 | + api.on(RooCodeEventName.Message, messageHandler) |
| 226 | + |
| 227 | + try { |
| 228 | + const taskId = await api.startNewTask({ |
| 229 | + configuration: { mode: "ask", alwaysAllowModeSwitch: true, autoApprovalEnabled: true }, |
| 230 | + text: `${promptTag}: what is 2+2? Reply with only the number.`, |
| 231 | + }) |
| 232 | + |
| 233 | + await waitUntilCompleted({ api, taskId }) |
| 234 | + } finally { |
| 235 | + api.off(RooCodeEventName.Message, messageHandler) |
| 236 | + } |
| 237 | + |
| 238 | + const firstRequest = requests.find((request) => request.lastUserMessage.includes(promptTag)) |
| 239 | + assert.ok(firstRequest, "Gemini provider should issue a generate content request for the task prompt") |
| 240 | + assert.strictEqual(firstRequest.model, GEMINI_MODEL_ID) |
| 241 | + assert.ok(firstRequest.hasTools, "Gemini provider should include tool declarations in the request") |
| 242 | + assert.ok( |
| 243 | + firstRequest.toolDeclarationCount > 0, |
| 244 | + "Gemini provider should declare at least one callable tool", |
| 245 | + ) |
| 246 | + assert.strictEqual( |
| 247 | + firstRequest.toolConfig?.functionCallingConfig?.allowedFunctionNames, |
| 248 | + undefined, |
| 249 | + "Gemini requests should not send allowedFunctionNames; the Gemini backend returns generic INVALID_ARGUMENT for larger or history-incompatible restriction lists", |
| 250 | + ) |
| 251 | + |
| 252 | + // Verify tool schemas are sanitized for Gemini compatibility. Gemini documents |
| 253 | + // function declaration schemas as a selected OpenAPI-style subset with |
| 254 | + // single-value `type` plus `nullable`; live testing also showed opaque |
| 255 | + // INVALID_ARGUMENT failures from broader third-party MCP schema metadata. |
| 256 | + for (const decl of firstRequest.functionDeclarations) { |
| 257 | + const violations = findInvalidSchemaPatterns( |
| 258 | + decl.parametersJsonSchema, |
| 259 | + `${decl.name}.parametersJsonSchema`, |
| 260 | + ) |
| 261 | + assert.strictEqual( |
| 262 | + violations.length, |
| 263 | + 0, |
| 264 | + `Tool "${decl.name}" has Gemini-incompatible schema: ${violations.join("; ")}`, |
| 265 | + ) |
| 266 | + } |
| 267 | + |
| 268 | + if (reasoningEffort === "disable") { |
| 269 | + assert.strictEqual( |
| 270 | + firstRequest.thinkingConfig, |
| 271 | + undefined, |
| 272 | + "Reasoning-disabled Gemini requests should omit thinkingConfig", |
| 273 | + ) |
| 274 | + } else { |
| 275 | + assert.ok( |
| 276 | + firstRequest.thinkingConfig, |
| 277 | + `Gemini requests with reasoningEffort="${reasoningEffort}" should include thinkingConfig`, |
| 278 | + ) |
| 279 | + } |
| 280 | + |
| 281 | + const completionMessage = messages.find( |
| 282 | + ({ say, text }) => (say === "completion_result" || say === "text") && text?.trim() === "4", |
| 283 | + ) |
| 284 | + |
| 285 | + assert.ok(completionMessage, "Task should complete with the expected Gemini provider response") |
| 286 | + }) |
| 287 | + } |
| 288 | +}) |
![roomote[bot]](https://avatars.githubusercontent.com/in/2924042?v=4&size=40){kind=avatar}
0 commit comments