Skip to content

Commit db07a23

Browse files
roomote[bot]roomoteedelauna
authored
[Improve] Show Zoo Code in outbound provider activity logs (#219)
* improve: update outbound request identity to Zoo Code * fix: address remaining outbound identity review feedback * fix: rename remaining Roo→Zoo identity strings * test(e2e): add outbound identity smoke tests for OpenRouter and Bedrock --------- Co-authored-by: Roomote <roomote@roocode.com> Co-authored-by: Elliott de Launay <edelauna@gmail.com>
1 parent 6470431 commit db07a23

32 files changed

Lines changed: 753 additions & 88 deletions
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "bedrock-identity-smoke"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "attempt_completion",
11+
"arguments": "{\"result\":\"4\"}",
12+
"id": "call_bedrock_identity_001"
13+
}
14+
]
15+
}
16+
}
17+
]
18+
}
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"fixtures": [
3+
{
4+
"match": {
5+
"userMessage": "openrouter-identity-smoke"
6+
},
7+
"response": {
8+
"toolCalls": [
9+
{
10+
"name": "attempt_completion",
11+
"arguments": "{\"result\":\"4\"}",
12+
"id": "call_openrouter_identity_001"
13+
}
14+
]
15+
}
16+
}
17+
]
18+
}
Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
import * as http2 from "http2"
2+
import * as net from "net"
3+
import { crc32 } from "zlib"
4+
5+
export interface BedrockMockServer {
6+
url: string
7+
/** Headers from the most recent converse-stream request (populated after first call). */
8+
lastRequestHeaders: http2.IncomingHttpHeaders | undefined
9+
close(): Promise<void>
10+
}
11+
12+
// AWS binary event stream encoder — matches the real Bedrock wire format.
13+
// Derived from the live converse-stream capture:
14+
// { contentBlockDelta: { contentBlockIndex: 0, delta: { toolUse: { input: "..." } } } }
15+
// aimock's builder nests the payload one level too deep (contentBlockDelta inside contentBlockDelta),
16+
// causing the AWS SDK deserializer's take() to miss the delta field entirely.
17+
function encodeHeaders(headers: Record<string, string>): Buffer {
18+
const parts: Buffer[] = []
19+
for (const [name, value] of Object.entries(headers)) {
20+
const nameBytes = Buffer.from(name, "utf8")
21+
const valueBytes = Buffer.from(value, "utf8")
22+
const buf = Buffer.alloc(1 + nameBytes.length + 1 + 2 + valueBytes.length)
23+
let off = 0
24+
buf.writeUInt8(nameBytes.length, off)
25+
off += 1
26+
nameBytes.copy(buf, off)
27+
off += nameBytes.length
28+
buf.writeUInt8(7, off)
29+
off += 1 // type 7 = string
30+
buf.writeUInt16BE(valueBytes.length, off)
31+
off += 2
32+
valueBytes.copy(buf, off)
33+
parts.push(buf)
34+
}
35+
return Buffer.concat(parts)
36+
}
37+
38+
function encodeFrame(eventType: string, payload: object): Buffer {
39+
const hdrs = encodeHeaders({
40+
":content-type": "application/json",
41+
":event-type": eventType,
42+
":message-type": "event",
43+
})
44+
const body = Buffer.from(JSON.stringify(payload), "utf8")
45+
const total = 12 + hdrs.length + body.length + 4
46+
const frame = Buffer.alloc(total)
47+
let off = 0
48+
frame.writeUInt32BE(total, off)
49+
off += 4
50+
frame.writeUInt32BE(hdrs.length, off)
51+
off += 4
52+
frame.writeUInt32BE(crc32(frame.subarray(0, 8)) >>> 0, off)
53+
off += 4
54+
hdrs.copy(frame, off)
55+
off += hdrs.length
56+
body.copy(frame, off)
57+
off += body.length
58+
frame.writeUInt32BE(crc32(frame.subarray(0, total - 4)) >>> 0, off)
59+
return frame
60+
}
61+
62+
function buildToolCallFrames(toolName: string, toolUseId: string, argsJson: string): Buffer[] {
63+
const frames: Buffer[] = []
64+
frames.push(encodeFrame("messageStart", { role: "assistant" }))
65+
frames.push(
66+
encodeFrame("contentBlockStart", {
67+
contentBlockIndex: 0,
68+
start: { toolUse: { name: toolName, toolUseId } },
69+
}),
70+
)
71+
const CHUNK = 20
72+
for (let i = 0; i < argsJson.length; i += CHUNK) {
73+
frames.push(
74+
encodeFrame("contentBlockDelta", {
75+
contentBlockIndex: 0,
76+
delta: { toolUse: { input: argsJson.slice(i, i + CHUNK) } },
77+
}),
78+
)
79+
}
80+
frames.push(encodeFrame("contentBlockStop", { contentBlockIndex: 0 }))
81+
frames.push(encodeFrame("messageStop", { stopReason: "tool_use" }))
82+
frames.push(
83+
encodeFrame("metadata", {
84+
metrics: { latencyMs: 1 },
85+
usage: { inputTokens: 100, outputTokens: 10, totalTokens: 110, serverToolUsage: {} },
86+
}),
87+
)
88+
return frames
89+
}
90+
91+
export async function startBedrockMockServer(): Promise<BedrockMockServer> {
92+
// HTTP/2 cleartext (h2c) — matches what @aws-sdk/client-bedrock-runtime uses by default.
93+
const server = http2.createServer()
94+
let lastRequestHeaders: http2.IncomingHttpHeaders | undefined
95+
const sessions = new Set<http2.ServerHttp2Session>()
96+
97+
server.on("session", (session) => {
98+
sessions.add(session)
99+
session.on("close", () => sessions.delete(session))
100+
})
101+
102+
server.on("stream", (stream, headers) => {
103+
const path = headers[":path"] as string
104+
const method = headers[":method"] as string
105+
106+
if (!path?.includes("converse-stream") || method !== "POST") {
107+
stream.respond({ ":status": 404 })
108+
stream.end(JSON.stringify({ error: { message: "Not found", type: "not_found" } }))
109+
return
110+
}
111+
112+
lastRequestHeaders = headers
113+
114+
// Drain the request body before responding (AWS SDK sends the full request before reading).
115+
stream.resume()
116+
stream.on("end", () => {
117+
stream.respond({
118+
":status": 200,
119+
"content-type": "application/vnd.amazon.eventstream",
120+
})
121+
const frames = buildToolCallFrames(
122+
"attempt_completion",
123+
"tooluse_bedrock_mock_001",
124+
JSON.stringify({ result: "4" }),
125+
)
126+
for (const frame of frames) {
127+
stream.write(frame)
128+
}
129+
stream.end()
130+
})
131+
})
132+
133+
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve))
134+
const addr = server.address() as net.AddressInfo
135+
const url = `http://127.0.0.1:${addr.port}`
136+
137+
return {
138+
url,
139+
get lastRequestHeaders() {
140+
return lastRequestHeaders
141+
},
142+
close: () => {
143+
// Destroy all open HTTP/2 sessions first so server.close() resolves immediately
144+
// instead of waiting for the extension's retry loop to exhaust itself.
145+
for (const session of sessions) {
146+
session.destroy()
147+
}
148+
sessions.clear()
149+
return new Promise<void>((resolve, reject) => server.close((err) => (err ? reject(err) : resolve())))
150+
},
151+
}
152+
}

apps/vscode-e2e/src/runTest.ts

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,21 @@ function isDeepSeekTargetedRun(testFile?: string, testGrep?: string) {
2626
return testGrep?.toLowerCase().includes("deepseek") ?? false
2727
}
2828

29+
function isBedrockTargetedRun(testFile?: string, testGrep?: string) {
30+
if (testFile?.toLowerCase().includes("bedrock.test")) {
31+
return true
32+
}
33+
34+
return testGrep?.toLowerCase().includes("bedrock") ?? false
35+
}
36+
2937
async function main() {
3038
const isRecord = process.env.AIMOCK_RECORD === "true"
3139
const testGrep = getCliFlagValue("--grep") || process.env.TEST_GREP
3240
const testFile = getCliFlagValue("--file") || process.env.TEST_FILE
3341
const isDeepSeekTest = isDeepSeekTargetedRun(testFile, testGrep)
3442
const isGeminiTest = testFile?.toLowerCase().includes("gemini.test") ?? false
43+
const isBedrockTest = isBedrockTargetedRun(testFile, testGrep)
3544

3645
if (isRecord && isDeepSeekTest && !process.env.DEEPSEEK_API_KEY) {
3746
throw new Error("AIMOCK_RECORD=true requires DEEPSEEK_API_KEY to record DeepSeek fixtures")
@@ -49,7 +58,9 @@ async function main() {
4958
// Replay mode starts aimock when no real API key is present or USE_MOCK is forced.
5059
const hasRealApiKey = isDeepSeekTest
5160
? !!process.env.DEEPSEEK_API_KEY
52-
: !!(process.env.OPENROUTER_API_KEY || process.env.ANTHROPIC_API_KEY)
61+
: isBedrockTest
62+
? true // Bedrock test starts its own binary-event-stream mock server when no real token
63+
: !!(process.env.OPENROUTER_API_KEY || process.env.ANTHROPIC_API_KEY)
5364
const useMock = isRecord || !hasRealApiKey || process.env.USE_MOCK === "true"
5465

5566
let mock: InstanceType<typeof LLMock> | undefined
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import * as assert from "assert"
2+
3+
import { startBedrockMockServer, type BedrockMockServer } from "../../bedrock-mock-server"
4+
import { setDefaultSuiteTimeout } from "../test-utils"
5+
import { waitUntilCompleted } from "../utils"
6+
7+
const AWS_BEARER_TOKEN_BEDROCK = process.env.AWS_BEARER_TOKEN_BEDROCK
8+
const BEDROCK_REGION = process.env.BEDROCK_REGION ?? "us-east-1"
9+
// Use a cross-region inference profile so the token works without per-region model access.
10+
const BEDROCK_MODEL_ID = process.env.BEDROCK_MODEL_ID ?? "us.anthropic.claude-haiku-4-5-20251001-v1:0"
11+
const BEDROCK_LIVE_E2E = process.env.BEDROCK_LIVE_E2E === "true"
12+
13+
suite("Bedrock provider", function () {
14+
setDefaultSuiteTimeout(this)
15+
this.timeout(3 * 60_000)
16+
17+
let mockServer: BedrockMockServer | undefined
18+
19+
suiteSetup(async function () {
20+
const aimockUrl = process.env.AIMOCK_URL
21+
22+
if (!aimockUrl && BEDROCK_LIVE_E2E && AWS_BEARER_TOKEN_BEDROCK) {
23+
// Live mode — explicitly opted into real AWS credentials, no aimock intercepting traffic.
24+
await globalThis.api.setConfiguration({
25+
apiProvider: "bedrock" as const,
26+
awsUseApiKey: true,
27+
awsApiKey: AWS_BEARER_TOKEN_BEDROCK,
28+
awsRegion: BEDROCK_REGION,
29+
apiModelId: BEDROCK_MODEL_ID,
30+
})
31+
} else {
32+
// Mock mode — use our custom binary-event-stream server because aimock's
33+
// converse-stream builder nests payloads one level too deep, causing the AWS SDK
34+
// deserializer to drop the delta field (take() reads top-level only).
35+
mockServer = await startBedrockMockServer()
36+
await globalThis.api.setConfiguration({
37+
apiProvider: "bedrock" as const,
38+
awsUseApiKey: true,
39+
awsApiKey: "mock-key",
40+
awsRegion: BEDROCK_REGION,
41+
apiModelId: BEDROCK_MODEL_ID,
42+
awsBedrockEndpoint: mockServer.url,
43+
awsBedrockEndpointEnabled: true,
44+
})
45+
}
46+
})
47+
48+
suiteTeardown(async () => {
49+
// Restore the default provider first so the extension stops using the Bedrock
50+
// endpoint. Only then close the mock server — closing it first leaves any
51+
// in-flight retry loop hitting ECONNREFUSED and the after-all hook times out.
52+
const aimockUrl = process.env.AIMOCK_URL
53+
const isRecord = process.env.AIMOCK_RECORD === "true"
54+
await globalThis.api.setConfiguration({
55+
apiProvider: "openrouter" as const,
56+
openRouterApiKey: aimockUrl && !isRecord ? "mock-key" : process.env.OPENROUTER_API_KEY!,
57+
openRouterModelId: "openai/gpt-4.1",
58+
...(aimockUrl && { openRouterBaseUrl: `${aimockUrl}/v1` }),
59+
})
60+
61+
if (mockServer) {
62+
// Brief pause so the extension picks up the provider switch before the
63+
// h2c listener closes.
64+
await new Promise<void>((resolve) => setTimeout(resolve, 500))
65+
await mockServer.close()
66+
mockServer = undefined
67+
}
68+
})
69+
70+
test("Should complete a task end-to-end via AWS Bedrock with ZooCode# user-agent", async () => {
71+
const api = globalThis.api
72+
const taskId = await api.startNewTask({
73+
configuration: { mode: "ask", autoApprovalEnabled: true },
74+
text: "bedrock-identity-smoke: what is 2+2? Reply with only the number.",
75+
})
76+
77+
await waitUntilCompleted({ api, taskId })
78+
79+
if (mockServer) {
80+
// Verify the AWS SDK transmitted the ZooCode# userAgentAppId.
81+
// In Node.js mode the SDK appends "app/<appId>" to the full sdkUserAgentValue
82+
// and writes it to the "user-agent" header. The "x-amz-user-agent" header only
83+
// carries aws-sdk-* segments and never contains the app ID.
84+
const userAgent = mockServer.lastRequestHeaders?.["user-agent"] as string | undefined
85+
assert.ok(userAgent, "Bedrock request should include user-agent header")
86+
assert.ok(userAgent.includes("ZooCode#"), `user-agent should contain "ZooCode#" — got: ${userAgent}`)
87+
} else {
88+
// Live mode: a successful round-trip proves the identity change didn't break
89+
// SDK auth or request formation. The x-amzn-user-agent header is not visible
90+
// to us without intercepting at the TLS layer.
91+
assert.ok(true, "Task completed successfully via Bedrock with ZooCode# userAgentAppId")
92+
}
93+
})
94+
})

0 commit comments

Comments
 (0)