Skip to content

Commit 7eda7e6

Browse files
committed
test(e2e): add outbound identity smoke tests for OpenRouter and Bedrock
1 parent ecd9623 commit 7eda7e6

6 files changed

Lines changed: 370 additions & 1 deletion

File tree

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "bedrock-identity-smoke"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "attempt_completion",
11+
"arguments": "{\"result\":\"4\"}",
12+
"id": "call_bedrock_identity_001"
13+
}
14+
]
15+
}
16+
}
17+
]
18+
}
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "openrouter-identity-smoke"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "attempt_completion",
11+
"arguments": "{\"result\":\"4\"}",
12+
"id": "call_openrouter_identity_001"
13+
}
14+
]
15+
}
16+
}
17+
]
18+
}
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
import * as http2 from "http2"
2+
import * as net from "net"
3+
import { crc32 } from "zlib"
4+
5+
export interface BedrockMockServer {
6+
url: string
7+
close(): Promise<void>
8+
}
9+
10+
// AWS binary event stream encoder — matches the real Bedrock wire format.
11+
// Derived from the live converse-stream capture:
12+
// { contentBlockDelta: { contentBlockIndex: 0, delta: { toolUse: { input: "..." } } } }
13+
// aimock's builder nests the payload one level too deep (contentBlockDelta inside contentBlockDelta),
14+
// causing the AWS SDK deserializer's take() to miss the delta field entirely.
15+
function encodeHeaders(headers: Record<string, string>): Buffer {
16+
const parts: Buffer[] = []
17+
for (const [name, value] of Object.entries(headers)) {
18+
const nameBytes = Buffer.from(name, "utf8")
19+
const valueBytes = Buffer.from(value, "utf8")
20+
const buf = Buffer.alloc(1 + nameBytes.length + 1 + 2 + valueBytes.length)
21+
let off = 0
22+
buf.writeUInt8(nameBytes.length, off)
23+
off += 1
24+
nameBytes.copy(buf, off)
25+
off += nameBytes.length
26+
buf.writeUInt8(7, off)
27+
off += 1 // type 7 = string
28+
buf.writeUInt16BE(valueBytes.length, off)
29+
off += 2
30+
valueBytes.copy(buf, off)
31+
parts.push(buf)
32+
}
33+
return Buffer.concat(parts)
34+
}
35+
36+
function encodeFrame(eventType: string, payload: object): Buffer {
37+
const hdrs = encodeHeaders({
38+
":content-type": "application/json",
39+
":event-type": eventType,
40+
":message-type": "event",
41+
})
42+
const body = Buffer.from(JSON.stringify(payload), "utf8")
43+
const total = 12 + hdrs.length + body.length + 4
44+
const frame = Buffer.alloc(total)
45+
let off = 0
46+
frame.writeUInt32BE(total, off)
47+
off += 4
48+
frame.writeUInt32BE(hdrs.length, off)
49+
off += 4
50+
frame.writeUInt32BE(crc32(frame.subarray(0, 8)) >>> 0, off)
51+
off += 4
52+
hdrs.copy(frame, off)
53+
off += hdrs.length
54+
body.copy(frame, off)
55+
off += body.length
56+
frame.writeUInt32BE(crc32(frame.subarray(0, total - 4)) >>> 0, off)
57+
return frame
58+
}
59+
60+
function buildToolCallFrames(toolName: string, toolUseId: string, argsJson: string): Buffer[] {
61+
const frames: Buffer[] = []
62+
frames.push(encodeFrame("messageStart", { role: "assistant" }))
63+
frames.push(
64+
encodeFrame("contentBlockStart", {
65+
contentBlockIndex: 0,
66+
start: { toolUse: { name: toolName, toolUseId } },
67+
}),
68+
)
69+
const CHUNK = 20
70+
for (let i = 0; i < argsJson.length; i += CHUNK) {
71+
frames.push(
72+
encodeFrame("contentBlockDelta", {
73+
contentBlockIndex: 0,
74+
delta: { toolUse: { input: argsJson.slice(i, i + CHUNK) } },
75+
}),
76+
)
77+
}
78+
frames.push(encodeFrame("contentBlockStop", { contentBlockIndex: 0 }))
79+
frames.push(encodeFrame("messageStop", { stopReason: "tool_use" }))
80+
frames.push(
81+
encodeFrame("metadata", {
82+
metrics: { latencyMs: 1 },
83+
usage: { inputTokens: 100, outputTokens: 10, totalTokens: 110, serverToolUsage: {} },
84+
}),
85+
)
86+
return frames
87+
}
88+
89+
export async function startBedrockMockServer(): Promise<BedrockMockServer> {
90+
// HTTP/2 cleartext (h2c) — matches what @aws-sdk/client-bedrock-runtime uses by default.
91+
const server = http2.createServer()
92+
93+
server.on("stream", (stream, headers) => {
94+
const path = headers[":path"] as string
95+
const method = headers[":method"] as string
96+
97+
if (!path?.includes("converse-stream") || method !== "POST") {
98+
stream.respond({ ":status": 404 })
99+
stream.end(JSON.stringify({ error: { message: "Not found", type: "not_found" } }))
100+
return
101+
}
102+
103+
// Drain the request body before responding (AWS SDK sends the full request before reading).
104+
stream.resume()
105+
stream.on("end", () => {
106+
stream.respond({
107+
":status": 200,
108+
"content-type": "application/vnd.amazon.eventstream",
109+
})
110+
const frames = buildToolCallFrames(
111+
"attempt_completion",
112+
"tooluse_bedrock_mock_001",
113+
JSON.stringify({ result: "4" }),
114+
)
115+
for (const frame of frames) {
116+
stream.write(frame)
117+
}
118+
stream.end()
119+
})
120+
})
121+
122+
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve))
123+
const addr = server.address() as net.AddressInfo
124+
125+
return {
126+
url: `http://127.0.0.1:${addr.port}`,
127+
close: () => new Promise<void>((resolve, reject) => server.close((err) => (err ? reject(err) : resolve()))),
128+
}
129+
}

apps/vscode-e2e/src/runTest.ts

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,21 @@ function isDeepSeekTargetedRun(testFile?: string, testGrep?: string) {
2626
return testGrep?.toLowerCase().includes("deepseek") ?? false
2727
}
2828

29+
function isBedrockTargetedRun(testFile?: string, testGrep?: string) {
30+
if (testFile?.toLowerCase().includes("bedrock.test")) {
31+
return true
32+
}
33+
34+
return testGrep?.toLowerCase().includes("bedrock") ?? false
35+
}
36+
2937
async function main() {
3038
const isRecord = process.env.AIMOCK_RECORD === "true"
3139
const testGrep = getCliFlagValue("--grep") || process.env.TEST_GREP
3240
const testFile = getCliFlagValue("--file") || process.env.TEST_FILE
3341
const isDeepSeekTest = isDeepSeekTargetedRun(testFile, testGrep)
3442
const isGeminiTest = testFile?.toLowerCase().includes("gemini.test") ?? false
43+
const isBedrockTest = isBedrockTargetedRun(testFile, testGrep)
3544

3645
if (isRecord && isDeepSeekTest && !process.env.DEEPSEEK_API_KEY) {
3746
throw new Error("AIMOCK_RECORD=true requires DEEPSEEK_API_KEY to record DeepSeek fixtures")
@@ -49,7 +58,9 @@ async function main() {
4958
// Replay mode starts aimock when no real API key is present or USE_MOCK is forced.
5059
const hasRealApiKey = isDeepSeekTest
5160
? !!process.env.DEEPSEEK_API_KEY
52-
: !!(process.env.OPENROUTER_API_KEY || process.env.ANTHROPIC_API_KEY)
61+
: isBedrockTest
62+
? true // Bedrock test starts its own binary-event-stream mock server when no real token
63+
: !!(process.env.OPENROUTER_API_KEY || process.env.ANTHROPIC_API_KEY)
5364
const useMock = isRecord || !hasRealApiKey || process.env.USE_MOCK === "true"
5465

5566
let mock: InstanceType<typeof LLMock> | undefined
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import * as assert from "assert"
2+
3+
import { startBedrockMockServer, type BedrockMockServer } from "../../bedrock-mock-server"
4+
import { setDefaultSuiteTimeout } from "../test-utils"
5+
import { waitUntilCompleted } from "../utils"
6+
7+
const AWS_BEARER_TOKEN_BEDROCK = process.env.AWS_BEARER_TOKEN_BEDROCK
8+
const BEDROCK_REGION = process.env.BEDROCK_REGION ?? "us-east-1"
9+
// Use a cross-region inference profile so the token works without per-region model access.
10+
const BEDROCK_MODEL_ID = process.env.BEDROCK_MODEL_ID ?? "us.anthropic.claude-haiku-4-5-20251001-v1:0"
11+
12+
suite("Bedrock provider", function () {
13+
setDefaultSuiteTimeout(this)
14+
this.timeout(3 * 60_000)
15+
16+
let mockServer: BedrockMockServer | undefined
17+
18+
suiteSetup(async function () {
19+
if (AWS_BEARER_TOKEN_BEDROCK) {
20+
// Live mode — real AWS credentials
21+
await globalThis.api.setConfiguration({
22+
apiProvider: "bedrock" as const,
23+
awsUseApiKey: true,
24+
awsApiKey: AWS_BEARER_TOKEN_BEDROCK,
25+
awsRegion: BEDROCK_REGION,
26+
apiModelId: BEDROCK_MODEL_ID,
27+
})
28+
} else {
29+
// Mock mode — use our custom binary-event-stream server because aimock's
30+
// converse-stream builder nests payloads one level too deep, causing the AWS SDK
31+
// deserializer to drop the delta field (take() reads top-level only).
32+
mockServer = await startBedrockMockServer()
33+
await globalThis.api.setConfiguration({
34+
apiProvider: "bedrock" as const,
35+
awsUseApiKey: true,
36+
awsApiKey: "mock-key",
37+
awsRegion: BEDROCK_REGION,
38+
apiModelId: BEDROCK_MODEL_ID,
39+
awsBedrockEndpoint: mockServer.url,
40+
awsBedrockEndpointEnabled: true,
41+
})
42+
}
43+
})
44+
45+
suiteTeardown(async () => {
46+
if (mockServer) {
47+
await mockServer.close()
48+
mockServer = undefined
49+
}
50+
const aimockUrl = process.env.AIMOCK_URL
51+
const isRecord = process.env.AIMOCK_RECORD === "true"
52+
await globalThis.api.setConfiguration({
53+
apiProvider: "openrouter" as const,
54+
openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!,
55+
openRouterModelId: "openai/gpt-4.1",
56+
...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }),
57+
})
58+
})
59+
60+
test("Should complete a task end-to-end via AWS Bedrock with ZooCode# user-agent", async () => {
61+
const api = globalThis.api
62+
const taskId = await api.startNewTask({
63+
configuration: { mode: "ask", autoApprovalEnabled: true },
64+
text: "bedrock-identity-smoke: what is 2+2? Reply with only the number.",
65+
})
66+
67+
// The AWS SDK uses Node.js http/https (not globalThis.fetch), so we
68+
// verify ZooCode# userAgentAppId indirectly: a successful round-trip proves the
69+
// identity change didn't break SDK auth or request formation.
70+
await waitUntilCompleted({ api, taskId })
71+
72+
assert.ok(true, "Task completed successfully via Bedrock with ZooCode# userAgentAppId")
73+
})
74+
})
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import * as assert from "assert"
2+
3+
import { setDefaultSuiteTimeout } from "../test-utils"
4+
import { waitUntilCompleted } from "../utils"
5+
6+
const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY
7+
8+
type CapturedOpenRouterRequest = {
9+
xTitle: string | undefined
10+
httpReferer: string | undefined
11+
userAgent: string | undefined
12+
}
13+
14+
function getRequestUrl(input: RequestInfo | URL): string {
15+
return typeof input === "string" ? input : input instanceof URL ? input.href : (input as Request).url
16+
}
17+
18+
function getHeaderValue(init: RequestInit | undefined, name: string): string | undefined {
19+
if (!init?.headers) return undefined
20+
const lower = name.toLowerCase()
21+
if (Array.isArray(init.headers)) {
22+
const found = (init.headers as string[][]).find(([k]) => k?.toLowerCase() === lower)
23+
return found?.[1]
24+
}
25+
if (init.headers instanceof Headers) {
26+
return init.headers.get(name) ?? undefined
27+
}
28+
const record = init.headers as Record<string, string>
29+
return record[name] ?? Object.entries(record).find(([k]) => k.toLowerCase() === lower)?.[1]
30+
}
31+
32+
function installOpenRouterRequestCapture(capture: CapturedOpenRouterRequest[], baseUrl: string): () => void {
33+
const originalFetch = globalThis.fetch
34+
const targetOrigin = new URL(baseUrl).origin
35+
36+
globalThis.fetch = async function (input: RequestInfo | URL, init?: RequestInit): Promise<Response> {
37+
const url = getRequestUrl(input)
38+
39+
try {
40+
if (new URL(url).origin === targetOrigin) {
41+
const xTitle = getHeaderValue(init, "X-Title") ?? getHeaderValue(init, "x-title")
42+
if (xTitle !== undefined) {
43+
capture.push({
44+
xTitle,
45+
httpReferer: getHeaderValue(init, "HTTP-Referer") ?? getHeaderValue(init, "http-referer"),
46+
userAgent: getHeaderValue(init, "User-Agent") ?? getHeaderValue(init, "user-agent"),
47+
})
48+
}
49+
}
50+
} catch {
51+
// ignore invalid URLs
52+
}
53+
54+
return originalFetch.call(globalThis, input, init as RequestInit)
55+
} as typeof globalThis.fetch
56+
57+
return () => {
58+
globalThis.fetch = originalFetch
59+
}
60+
}
61+
62+
suite("OpenRouter provider", function () {
63+
setDefaultSuiteTimeout(this)
64+
65+
let restoreFetch: (() => void) | undefined
66+
const requests: CapturedOpenRouterRequest[] = []
67+
68+
setup(function () {
69+
const aimockUrl = process.env.AIMOCK_URL
70+
if (!aimockUrl && !OPENROUTER_API_KEY) {
71+
this.skip()
72+
}
73+
})
74+
75+
suiteSetup(async () => {
76+
const aimockUrl = process.env.AIMOCK_URL
77+
const isRecord = process.env.AIMOCK_RECORD === "true"
78+
const baseUrl = aimockUrl ? `${aimockUrl}/v1` : "https://openrouter.ai/api/v1"
79+
80+
restoreFetch = installOpenRouterRequestCapture(requests, baseUrl)
81+
82+
await globalThis.api.setConfiguration({
83+
apiProvider: "openrouter" as const,
84+
openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : OPENROUTER_API_KEY!,
85+
openRouterModelId: "openai/gpt-4.1",
86+
...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }),
87+
})
88+
})
89+
90+
suiteTeardown(() => {
91+
restoreFetch?.()
92+
restoreFetch = undefined
93+
})
94+
95+
test("Should identify as Zoo Code in outbound DEFAULT_HEADERS", async () => {
96+
requests.length = 0
97+
98+
const api = globalThis.api
99+
const taskId = await api.startNewTask({
100+
configuration: { mode: "ask", autoApprovalEnabled: true },
101+
text: "openrouter-identity-smoke: what is 2+2? Reply with only the number.",
102+
})
103+
104+
await waitUntilCompleted({ api, taskId })
105+
106+
const captured = requests[0]
107+
assert.ok(captured, "OpenRouter provider should issue at least one outbound request")
108+
assert.strictEqual(captured.xTitle, "Zoo Code", "X-Title header should identify the extension as Zoo Code")
109+
assert.strictEqual(
110+
captured.httpReferer,
111+
"https://github.com/Zoo-Code-Org/Zoo-Code",
112+
"HTTP-Referer header should point to the Zoo Code repository",
113+
)
114+
assert.ok(
115+
captured.userAgent?.startsWith("ZooCode/"),
116+
`User-Agent should start with "ZooCode/" — got: ${captured.userAgent}`,
117+
)
118+
})
119+
})

0 commit comments

Comments
 (0)