Skip to content

Commit 121762c

Browse files
edelaunaedelauna
committed
test(gemini-e2e): wire aimock recording and use real model id
1 parent 7ea196e commit 121762c

5 files changed

Lines changed: 56 additions & 7 deletions

File tree

‎apps/vscode-e2e/AGENTS.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,26 @@ ZAI_API_KEY=<key> TEST_FILE=zai.test pnpm --filter @roo-code/vscode-e2e test:ci
160160
161161
When adding a new test to this suite, add a matching fixture to the `installZAiFetchInterceptor` call in `suiteSetup`. Use a short unique prefix (e.g. `"zai-glm-e2e-mytest:"`) that won't appear in `<environment_details>`.
162162
163+
### Gemini (`suite/providers/gemini.test.ts`)
164+
165+
Gemini routes through aimock via `googleGeminiBaseUrl: aimockUrl`. aimock has native Gemini SSE support and can proxy to `https://generativelanguage.googleapis.com` in record mode. The model ID defaults to `gemini-3-flash-preview` but can be overridden via `GEMINI_MODEL_ID`.
166+
167+
The test only runs when aimock is active (replay or record). Live runs without aimock are not supported because `GEMINI_MODEL_ID` must match the fixture.
168+
169+
**Record** (refresh fixtures from the real Gemini API):
170+
171+
```sh
172+
GEMINI_API_KEY=<key> TEST_FILE=providers/gemini.test pnpm --filter @roo-code/vscode-e2e test:record
173+
```
174+
175+
After recording, inspect the generated `fixtures/gemini-*.json`, extract the response blocks into `fixtures/gemini.json`, then delete the raw files.
176+
177+
**Verify in mock mode** (no key needed):
178+
179+
```sh
180+
TEST_FILE=providers/gemini.test pnpm --filter @roo-code/vscode-e2e test:ci:mock
181+
```
182+
163183
### xAI Grok (`suite/providers/xai.test.ts`)
164184
165185
xAI uses the **Responses API** (`POST https://api.x.ai/v1/responses`), which is not OpenAI-compatible. aimock can't intercept it. The suite instead patches `globalThis.fetch` to intercept requests to that endpoint. By default it replays hand-crafted SSE events; when a local `fixtures/xai.json` recording exists, it can replay recorded real-API SSE events for reference.

‎apps/vscode-e2e/fixtures/gemini.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"fixtures": [
33
{
44
"match": {
5-
"model": "gemini-3.1-pro-preview",
5+
"model": "gemini-3-flash-preview",
66
"userMessage": "gemini-e2e:reasoning-high: what is 2+2? Reply with only the number."
77
},
88
"response": {
@@ -17,7 +17,7 @@
1717
},
1818
{
1919
"match": {
20-
"model": "gemini-3.1-pro-preview",
20+
"model": "gemini-3-flash-preview",
2121
"userMessage": "gemini-e2e:reasoning-low: what is 2+2? Reply with only the number."
2222
},
2323
"response": {
@@ -32,7 +32,7 @@
3232
},
3333
{
3434
"match": {
35-
"model": "gemini-3.1-pro-preview",
35+
"model": "gemini-3-flash-preview",
3636
"userMessage": "gemini-e2e:reasoning-disable: what is 2+2? Reply with only the number."
3737
},
3838
"response": {

‎apps/vscode-e2e/src/runTest.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,17 @@ async function main() {
3030
const testGrep = getCliFlagValue("--grep") || process.env.TEST_GREP
3131
const testFile = getCliFlagValue("--file") || process.env.TEST_FILE
3232
const isDeepSeekTest = isDeepSeekTargetedRun(testFile, testGrep)
33+
const isGeminiTest = testFile?.toLowerCase().includes("gemini.test") ?? false
3334

3435
if (isRecord && isDeepSeekTest && !process.env.DEEPSEEK_API_KEY) {
3536
throw new Error("AIMOCK_RECORD=true requires DEEPSEEK_API_KEY to record DeepSeek fixtures")
3637
}
3738

38-
if (isRecord && !isDeepSeekTest && !process.env.OPENROUTER_API_KEY) {
39+
if (isRecord && isGeminiTest && !process.env.GEMINI_API_KEY && !process.env.GOOGLE_API_KEY) {
40+
throw new Error("AIMOCK_RECORD=true requires GEMINI_API_KEY to record Gemini fixtures")
41+
}
42+
43+
if (isRecord && !isDeepSeekTest && !isGeminiTest && !process.env.OPENROUTER_API_KEY) {
3944
throw new Error("AIMOCK_RECORD=true requires OPENROUTER_API_KEY to record fixtures")
4045
}
4146

@@ -73,6 +78,8 @@ async function main() {
7378
openai: isDeepSeekTest ? "https://api.deepseek.com" : "https://openrouter.ai/api",
7479
// aimock forwards the x-api-key header from the Anthropic SDK to the real API.
7580
anthropic: "https://api.anthropic.com",
81+
// aimock forwards the x-goog-api-key header from the Google AI SDK.
82+
...(isGeminiTest && { gemini: "https://generativelanguage.googleapis.com" }),
7683
},
7784
fixturePath: fixturesDir,
7885
},

‎apps/vscode-e2e/src/suite/providers/gemini.test.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { setDefaultSuiteTimeout } from "../test-utils"
66
import { waitUntilCompleted } from "../utils"
77

88
const GEMINI_API_KEY = process.env.GEMINI_API_KEY ?? process.env.GOOGLE_API_KEY
9-
const GEMINI_MODEL_ID = "gemini-3.1-pro-preview"
9+
const GEMINI_MODEL_ID = process.env.GEMINI_MODEL_ID ?? "gemini-3-flash-preview"
1010

1111
type FunctionDeclaration = {
1212
name: string
@@ -167,8 +167,11 @@ suite("Gemini provider", function () {
167167
const requests: CapturedGeminiRequest[] = []
168168

169169
setup(function () {
170-
const isReplay = process.env.AIMOCK_URL && process.env.AIMOCK_RECORD !== "true"
171-
if (!isReplay && !GEMINI_API_KEY) {
170+
const aimockUrl = process.env.AIMOCK_URL
171+
const isReplay = aimockUrl && process.env.AIMOCK_RECORD !== "true"
172+
const isRecordRun = aimockUrl && process.env.AIMOCK_RECORD === "true" && !!GEMINI_API_KEY
173+
// Live runs without aimock are not supported — GEMINI_MODEL_ID must match the fixture.
174+
if (!isReplay && !isRecordRun) {
172175
this.skip()
173176
}
174177
})

‎src/api/providers/__tests__/gemini-handler.spec.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ describe("GeminiHandler backend support", () => {
3030
} as ApiHandlerOptions
3131
const handler = new GeminiHandler(options)
3232
const stub = vi.fn().mockReturnValue((async function* () {})())
33+
// @ts-ignore access private client
3334
handler["client"].models.generateContentStream = stub
3435
await handler.createMessage("instr", [] as any).next()
3536
const config = stub.mock.calls[0][0].config
@@ -46,6 +47,7 @@ describe("GeminiHandler backend support", () => {
4647
} as ApiHandlerOptions
4748
const handler = new GeminiHandler(options)
4849
const stub = vi.fn().mockResolvedValue({ text: "ok" })
50+
// @ts-ignore access private client
4951
handler["client"].models.generateContent = stub
5052
const res = await handler.completePrompt("hi")
5153
expect(res).toBe("ok")
@@ -76,6 +78,7 @@ describe("GeminiHandler backend support", () => {
7678
}
7779

7880
const stub = vi.fn().mockReturnValue(mockStream())
81+
// @ts-ignore access private client
7982
handler["client"].models.generateContentStream = stub
8083

8184
const messages = []
@@ -114,6 +117,7 @@ describe("GeminiHandler backend support", () => {
114117
}
115118

116119
const stub = vi.fn().mockReturnValue(mockStream())
120+
// @ts-ignore access private client
117121
handler["client"].models.generateContentStream = stub
118122

119123
const messages = []
@@ -148,6 +152,7 @@ describe("GeminiHandler backend support", () => {
148152

149153
const mockError = new Error("API rate limit exceeded")
150154
const stub = vi.fn().mockRejectedValue(mockError)
155+
// @ts-ignore access private client
151156
handler["client"].models.generateContentStream = stub
152157

153158
await expect(async () => {
@@ -191,6 +196,7 @@ describe("GeminiHandler backend support", () => {
191196
} as ApiHandlerOptions
192197
const handler = new GeminiHandler(options)
193198
const stub = vi.fn().mockReturnValue((async function* () {})())
199+
// @ts-ignore access private client
194200
handler["client"].models.generateContentStream = stub
195201

196202
await handler
@@ -211,6 +217,7 @@ describe("GeminiHandler backend support", () => {
211217
} as ApiHandlerOptions
212218
const handler = new GeminiHandler(options)
213219
const stub = vi.fn().mockReturnValue((async function* () {})())
220+
// @ts-ignore access private client
214221
handler["client"].models.generateContentStream = stub
215222

216223
await handler
@@ -233,6 +240,7 @@ describe("GeminiHandler backend support", () => {
233240
} as ApiHandlerOptions
234241
const handler = new GeminiHandler(options)
235242
const stub = vi.fn().mockReturnValue((async function* () {})())
243+
// @ts-ignore access private client
236244
handler["client"].models.generateContentStream = stub
237245

238246
const manyTools = Array.from({ length: 30 }, (_, index) => ({
@@ -263,6 +271,7 @@ describe("GeminiHandler backend support", () => {
263271
} as ApiHandlerOptions
264272
const handler = new GeminiHandler(options)
265273
const stub = vi.fn().mockReturnValue((async function* () {})())
274+
// @ts-ignore access private client
266275
handler["client"].models.generateContentStream = stub
267276

268277
const manyTools = Array.from({ length: 30 }, (_, index) => ({
@@ -299,6 +308,7 @@ describe("GeminiHandler backend support", () => {
299308
} as ApiHandlerOptions
300309
const handler = new GeminiHandler(options)
301310
const stub = vi.fn().mockReturnValue((async function* () {})())
311+
// @ts-ignore access private client
302312
handler["client"].models.generateContentStream = stub
303313

304314
await handler
@@ -321,6 +331,7 @@ describe("GeminiHandler backend support", () => {
321331
} as ApiHandlerOptions
322332
const handler = new GeminiHandler(options)
323333
const stub = vi.fn().mockReturnValue((async function* () {})())
334+
// @ts-ignore access private client
324335
handler["client"].models.generateContentStream = stub
325336

326337
await handler
@@ -344,6 +355,7 @@ describe("GeminiHandler backend support", () => {
344355
} as ApiHandlerOptions
345356
const handler = new GeminiHandler(options)
346357
const stub = vi.fn().mockReturnValue((async function* () {})())
358+
// @ts-ignore access private client
347359
handler["client"].models.generateContentStream = stub
348360

349361
await handler
@@ -366,6 +378,7 @@ describe("GeminiHandler backend support", () => {
366378
} as ApiHandlerOptions
367379
const handler = new GeminiHandler(options)
368380
const stub = vi.fn().mockReturnValue((async function* () {})())
381+
// @ts-ignore access private client
369382
handler["client"].models.generateContentStream = stub
370383

371384
await handler
@@ -426,6 +439,7 @@ describe("GeminiHandler backend support", () => {
426439
} as ApiHandlerOptions
427440
const handler = new GeminiHandler(options)
428441
const stub = vi.fn().mockReturnValue((async function* () {})())
442+
// @ts-ignore access private client
429443
handler["client"].models.generateContentStream = stub
430444

431445
await handler
@@ -484,6 +498,7 @@ describe("GeminiHandler backend support", () => {
484498
const options = { apiProvider: "gemini" } as ApiHandlerOptions
485499
const handler = new GeminiHandler(options)
486500
const stub = vi.fn().mockReturnValue((async function* () {})())
501+
// @ts-ignore access private client
487502
handler["client"].models.generateContentStream = stub
488503

489504
await handler
@@ -528,6 +543,7 @@ describe("GeminiHandler backend support", () => {
528543
const options = { apiProvider: "gemini" } as ApiHandlerOptions
529544
const handler = new GeminiHandler(options)
530545
const stub = vi.fn().mockReturnValue((async function* () {})())
546+
// @ts-ignore access private client
531547
handler["client"].models.generateContentStream = stub
532548

533549
await handler
@@ -576,6 +592,7 @@ describe("GeminiHandler backend support", () => {
576592
const options = { apiProvider: "gemini" } as ApiHandlerOptions
577593
const handler = new GeminiHandler(options)
578594
const stub = vi.fn().mockReturnValue((async function* () {})())
595+
// @ts-ignore access private client
579596
handler["client"].models.generateContentStream = stub
580597

581598
await handler
@@ -617,6 +634,7 @@ describe("GeminiHandler backend support", () => {
617634
const options = { apiProvider: "gemini" } as ApiHandlerOptions
618635
const handler = new GeminiHandler(options)
619636
const stub = vi.fn().mockReturnValue((async function* () {})())
637+
// @ts-ignore access private client
620638
handler["client"].models.generateContentStream = stub
621639

622640
await handler
@@ -668,6 +686,7 @@ describe("GeminiHandler backend support", () => {
668686
const options = { apiProvider: "gemini" } as ApiHandlerOptions
669687
const handler = new GeminiHandler(options)
670688
const stub = vi.fn().mockReturnValue((async function* () {})())
689+
// @ts-ignore access private client
671690
handler["client"].models.generateContentStream = stub
672691

673692
await handler

0 commit comments

Comments
 (0)