diff --git a/CHANGELOG.md b/CHANGELOG.md index 949f9ba1..b5cff9e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # @copilotkit/aimock +## [1.16.2] - 2026-04-28 + +### Fixed + +- **Bedrock invoke native stream format** — `invoke-with-response-stream` now emits Anthropic-native snake_case payloads (`content_block_delta`, `input_json_delta`) wrapped in Bedrock EventStream `chunk` frames, instead of Converse-style camelCase events. Converse-stream retains camelCase format. (PR #144, sf-jin-ku) +- **Bedrock invoke false-green test** — Reasoning negative test used wrong event filters, masking a real bug; corrected to match actual stream shape (PR #144) +- **Bedrock invoke/stream hardening** — Set `completionReq.stream = true` in streaming handler; use deterministic `tool_use_${index}` fallback IDs; change `textContent || null` to `?? null` to preserve empty strings; warn on unsupported content block types and unexpected roles; add webSearches warning on tool-call-only responses +- **Converse stream shape alignment** — Wrap `contentBlockStop` and `messageStop` payloads to match real AWS Converse API; remove duplicate top-level `contentBlockIndex` from `contentBlockStart`/`contentBlockDelta`; add trailing `metadata` events (usage + latencyMs) to all stream builders +- **Converse request conversion** — Filter empty-string text blocks in all paths; unwrap `inputSchema` from `{ json: {...} }` Converse API wrapper; set `completionReq.stream = true` in streaming handler; add content-loss warnings for non-text blocks; fix error type `||` to `??` + +### Changed + +- Extract shared test helpers (`createMockReq`/`createMockRes`/`createDefaults`) into `helpers/mock-res.ts` +- Convert reasoning-all-providers tests to per-test server lifecycle +- Add content+toolCalls streaming integration coverage for both invoke and converse paths (PR #144) + ## [1.16.1] - 2026-04-28 ### Fixed diff --git a/charts/aimock/Chart.yaml b/charts/aimock/Chart.yaml index 04c53354..58f22fe4 100644 --- a/charts/aimock/Chart.yaml +++ b/charts/aimock/Chart.yaml @@ -3,4 +3,4 @@ name: aimock description: Mock infrastructure for AI application testing (OpenAI, Anthropic, Gemini, MCP, A2A, vector) type: application version: 0.1.0 -appVersion: "1.16.1" +appVersion: "1.16.2" diff --git a/package.json b/package.json index 71472af1..93c0cc23 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@copilotkit/aimock", - "version": "1.16.1", + "version": "1.16.2", "description": "Mock infrastructure for AI application testing — LLM APIs, image generation, text-to-speech, transcription, video generation, MCP tools, A2A agents, AG-UI event streams, vector databases, search, rerank, and moderation. One package, one port, zero dependencies.", "license": "MIT", "keywords": [ diff --git a/src/__tests__/bedrock-stream.test.ts b/src/__tests__/bedrock-stream.test.ts index db75909e..57e541b5 100644 --- a/src/__tests__/bedrock-stream.test.ts +++ b/src/__tests__/bedrock-stream.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect, afterEach } from "vitest"; import * as http from "node:http"; import { crc32 } from "node:zlib"; -import type { Fixture, HandlerDefaults } from "../types.js"; +import type { Fixture } from "../types.js"; import { createServer, type ServerInstance } from "../server.js"; import { converseToCompletionRequest, @@ -9,7 +9,7 @@ import { handleConverseStream, } from "../bedrock-converse.js"; import { Journal } from "../journal.js"; -import { Logger } from "../logger.js"; +import { createMockReq, createMockRes, createDefaults } from "./helpers/mock-res.js"; // --- helpers --- @@ -161,6 +161,13 @@ function postPartialBinary( const parsed = new URL(url); const chunks: Buffer[] = []; let aborted = false; + let resolved = false; + const safeResolve = (value: { body: Buffer; aborted: boolean }) => { + if (!resolved) { + resolved = true; + resolve(value); + } + }; const req = http.request( { hostname: parsed.hostname, @@ -175,7 +182,7 @@ function postPartialBinary( (res) => { res.on("data", (c: Buffer) => chunks.push(c)); res.on("end", () => { - resolve({ body: Buffer.concat(chunks), aborted }); + safeResolve({ body: Buffer.concat(chunks), aborted }); }); res.on("error", () => { aborted = true; @@ -184,13 +191,13 @@ function postPartialBinary( aborted = true; }); res.on("close", () => { - resolve({ body: Buffer.concat(chunks), aborted }); + safeResolve({ body: Buffer.concat(chunks), aborted }); }); }, ); req.on("error", () => { aborted = true; - resolve({ body: Buffer.concat(chunks), aborted }); + safeResolve({ body: Buffer.concat(chunks), aborted }); }); req.write(data); req.end(); @@ -775,7 +782,7 @@ describe("POST /model/{modelId}/converse-stream", () => { expect(fullText).toBe("Hi there!"); const msgStop = frames.find((f) => f.eventType === "messageStop"); - expect(msgStop!.payload).toEqual({ stopReason: "end_turn" }); + expect(msgStop!.payload).toEqual({ messageStop: { stopReason: "end_turn" } }); }); it("returns tool call response as Event Stream", async () => { @@ -810,7 +817,7 @@ describe("POST /model/{modelId}/converse-stream", () => { expect(JSON.parse(fullJson)).toEqual({ city: "SF" }); const msgStop = frames.find((f) => f.eventType === "messageStop"); - expect(msgStop!.payload).toEqual({ stopReason: "tool_use" }); + expect(msgStop!.payload).toEqual({ messageStop: { stopReason: "tool_use" } }); }); it("supports streaming profile (ttft/tps)", async () => { @@ -946,7 +953,240 @@ describe("POST /model/{modelId}/converse-stream (content + toolCalls)", () => { // messageStop with tool_use stop reason const msgStop = frames.find((f) => f.eventType === "messageStop"); - expect(msgStop!.payload).toEqual({ stopReason: "tool_use" }); + expect(msgStop!.payload).toEqual({ messageStop: { stopReason: "tool_use" } }); + }); +}); + +// ─── converse-stream: contentBlockStop wrapper shape ────────────────────────── + +describe("POST /model/{modelId}/converse-stream (contentBlockStop wrapper shape)", () => { + const MODEL_ID = "anthropic.claude-3-5-sonnet-20241022-v2:0"; + + it("contentBlockStop events have wrapped { contentBlockStop: { contentBlockIndex: N } } shape", async () => { + instance = await createServer(allFixtures); + const res = await postBinary(`${instance.url}/model/${MODEL_ID}/converse-stream`, { + messages: [{ role: "user", content: [{ text: "hello" }] }], + }); + + expect(res.status).toBe(200); + const frames = parseFrames(res.body); + + const stopFrames = frames.filter((f) => f.eventType === "contentBlockStop"); + expect(stopFrames.length).toBeGreaterThanOrEqual(1); + + for (const frame of stopFrames) { + // Must be the wrapped shape, not the flat { contentBlockIndex: N } + const payload = frame.payload as { contentBlockStop: { contentBlockIndex: number } }; + expect(payload).toHaveProperty("contentBlockStop"); + expect(payload.contentBlockStop).toHaveProperty("contentBlockIndex"); + expect(typeof payload.contentBlockStop.contentBlockIndex).toBe("number"); + // Must NOT have a top-level contentBlockIndex (that would be the flat shape) + expect(Object.keys(payload)).toEqual(["contentBlockStop"]); + } + }); + + it("tool-call contentBlockStop events also have the wrapped shape", async () => { + instance = await createServer(allFixtures); + const res = await postBinary(`${instance.url}/model/${MODEL_ID}/converse-stream`, { + messages: [{ role: "user", content: [{ text: "weather" }] }], + }); + + expect(res.status).toBe(200); + const frames = parseFrames(res.body); + + const stopFrames = frames.filter((f) => f.eventType === "contentBlockStop"); + expect(stopFrames.length).toBeGreaterThanOrEqual(1); + + for (const frame of stopFrames) { + const payload = frame.payload as { contentBlockStop: { contentBlockIndex: number } }; + expect(payload).toHaveProperty("contentBlockStop"); + expect(payload.contentBlockStop).toHaveProperty("contentBlockIndex"); + expect(Object.keys(payload)).toEqual(["contentBlockStop"]); + } + }); + + it("messageStop events have the wrapped { messageStop: { stopReason: '...' } } shape", async () => { + instance = await createServer(allFixtures); + const res = await postBinary(`${instance.url}/model/${MODEL_ID}/converse-stream`, { + messages: [{ role: "user", content: [{ text: "hello" }] }], + }); + + expect(res.status).toBe(200); + const frames = parseFrames(res.body); + + const msgStopFrames = frames.filter((f) => f.eventType === "messageStop"); + expect(msgStopFrames).toHaveLength(1); + + const payload = msgStopFrames[0].payload as { messageStop: { stopReason: string } }; + expect(payload).toHaveProperty("messageStop"); + expect(payload.messageStop).toHaveProperty("stopReason"); + expect(Object.keys(payload)).toEqual(["messageStop"]); + }); +}); + +// ─── converse-stream: contentWithToolCalls full structure ───────────────────── + +describe("POST /model/{modelId}/converse-stream (contentWithToolCalls full structure)", () => { + const MODEL_ID = "anthropic.claude-3-5-sonnet-20241022-v2:0"; + + it("verifies complete event sequence for content + tool calls", async () => { + instance = await createServer(allFixtures); + const res = await postBinary(`${instance.url}/model/${MODEL_ID}/converse-stream`, { + messages: [{ role: "user", content: [{ text: "search-and-explain" }] }], + }); + + expect(res.status).toBe(200); + const frames = parseFrames(res.body); + + // 1. Stream starts with messageStart (role: assistant) + expect(frames[0].eventType).toBe("messageStart"); + expect(frames[0].payload).toEqual({ messageStart: { role: "assistant" } }); + + // 2. Collect all contentBlockStart frames + const blockStarts = frames.filter((f) => f.eventType === "contentBlockStart"); + expect(blockStarts.length).toBe(2); // one text, one tool + + // 3. Text content block appears before tool call block + const textBlockStartIdx = frames.findIndex( + (f) => + f.eventType === "contentBlockStart" && + (f.payload as { contentBlockStart?: { start?: { type?: string } } }).contentBlockStart + ?.start?.type === "text", + ); + const toolBlockStartIdx = frames.findIndex( + (f) => + f.eventType === "contentBlockStart" && + (f.payload as { contentBlockStart?: { start?: { toolUse?: unknown } } }).contentBlockStart + ?.start?.toolUse !== undefined, + ); + expect(textBlockStartIdx).toBeLessThan(toolBlockStartIdx); + + // 4. Tool call block has contentBlockStart with toolUse (toolUseId + name) + const toolBlockStart = frames[toolBlockStartIdx]; + const toolStartPayload = toolBlockStart.payload as { + contentBlockStart: { + contentBlockIndex: number; + start: { toolUse: { toolUseId: string; name: string } }; + }; + }; + expect(toolStartPayload.contentBlockStart.start.toolUse.name).toBe("web_search"); + expect(toolStartPayload.contentBlockStart.start.toolUse.toolUseId).toBeDefined(); + expect(typeof toolStartPayload.contentBlockStart.start.toolUse.toolUseId).toBe("string"); + + // 5. Tool call block has contentBlockDelta chunks after its start + const toolBlockIndex = toolStartPayload.contentBlockStart.contentBlockIndex; + const toolDeltas = frames.filter( + (f) => + f.eventType === "contentBlockDelta" && + (f.payload as { contentBlockDelta?: { contentBlockIndex?: number } }).contentBlockDelta + ?.contentBlockIndex === toolBlockIndex, + ); + expect(toolDeltas.length).toBeGreaterThanOrEqual(1); + + // 6. Tool call block has contentBlockStop + const toolBlockStop = frames.find( + (f) => + f.eventType === "contentBlockStop" && + (f.payload as { contentBlockStop?: { contentBlockIndex?: number } }).contentBlockStop + ?.contentBlockIndex === toolBlockIndex, + ); + expect(toolBlockStop).toBeDefined(); + expect(toolBlockStop!.payload).toEqual({ + contentBlockStop: { contentBlockIndex: toolBlockIndex }, + }); + + // 7. Stream ends with messageStop (stopReason: tool_use) then metadata + const msgStopIdx = frames.findIndex((f) => f.eventType === "messageStop"); + const metadataIdx = frames.findIndex((f) => f.eventType === "metadata"); + expect(msgStopIdx).toBeGreaterThan(-1); + expect(metadataIdx).toBeGreaterThan(-1); + expect(metadataIdx).toBe(msgStopIdx + 1); // metadata immediately follows messageStop + expect(metadataIdx).toBe(frames.length - 1); // metadata is last frame + + const msgStopPayload = frames[msgStopIdx].payload as { + messageStop: { stopReason: string }; + }; + expect(msgStopPayload).toEqual({ messageStop: { stopReason: "tool_use" } }); + + // 8. contentBlockIndex values are sequential across text and tool blocks + const allBlockStarts = frames + .filter((f) => f.eventType === "contentBlockStart") + .map( + (f) => + (f.payload as { contentBlockStart: { contentBlockIndex: number } }).contentBlockStart + .contentBlockIndex, + ); + expect(allBlockStarts).toEqual([0, 1]); + + const allBlockStops = frames + .filter((f) => f.eventType === "contentBlockStop") + .map( + (f) => + (f.payload as { contentBlockStop: { contentBlockIndex: number } }).contentBlockStop + .contentBlockIndex, + ); + expect(allBlockStops).toEqual([0, 1]); + }); + + it("verifies sequential contentBlockIndex with multiple tool calls", async () => { + const multiToolContentFixture: Fixture = { + match: { userMessage: "multi-tool-with-text" }, + response: { + content: "I will use two tools.", + toolCalls: [ + { name: "tool_a", arguments: '{"x":1}' }, + { name: "tool_b", arguments: '{"y":2}' }, + ], + }, + }; + instance = await createServer([multiToolContentFixture]); + const res = await postBinary(`${instance.url}/model/${MODEL_ID}/converse-stream`, { + messages: [{ role: "user", content: [{ text: "multi-tool-with-text" }] }], + }); + + expect(res.status).toBe(200); + const frames = parseFrames(res.body); + + // contentBlockIndex: 0 = text, 1 = tool_a, 2 = tool_b + const blockStarts = frames.filter((f) => f.eventType === "contentBlockStart"); + expect(blockStarts).toHaveLength(3); + + const indices = blockStarts.map( + (f) => + (f.payload as { contentBlockStart: { contentBlockIndex: number } }).contentBlockStart + .contentBlockIndex, + ); + expect(indices).toEqual([0, 1, 2]); + + // Text block at index 0 + const textStart = blockStarts[0].payload as { + contentBlockStart: { start: { type: string } }; + }; + expect(textStart.contentBlockStart.start.type).toBe("text"); + + // Tool blocks at indices 1 and 2 + const tool1Start = blockStarts[1].payload as { + contentBlockStart: { start: { toolUse: { name: string } } }; + }; + expect(tool1Start.contentBlockStart.start.toolUse.name).toBe("tool_a"); + + const tool2Start = blockStarts[2].payload as { + contentBlockStart: { start: { toolUse: { name: string } } }; + }; + expect(tool2Start.contentBlockStart.start.toolUse.name).toBe("tool_b"); + + // contentBlockStop indices are also sequential + const blockStops = frames.filter((f) => f.eventType === "contentBlockStop"); + const stopIndices = blockStops.map( + (f) => + (f.payload as { contentBlockStop: { contentBlockIndex: number } }).contentBlockStop + .contentBlockIndex, + ); + expect(stopIndices).toEqual([0, 1, 2]); + + // messageStop with tool_use + const msgStop = frames.find((f) => f.eventType === "messageStop"); + expect(msgStop!.payload).toEqual({ messageStop: { stopReason: "tool_use" } }); }); }); @@ -1472,7 +1712,7 @@ describe("converseToCompletionRequest (edge cases)", () => { }, "model", ); - expect(result.messages[0]).toEqual({ role: "assistant", content: null }); + expect(result.messages[0]).toEqual({ role: "assistant", content: "" }); }); it("handles user tool result with missing text in content items (text ?? '' fallback)", () => { @@ -1583,8 +1823,8 @@ describe("converseToCompletionRequest (edge cases)", () => { "model", ); expect(result.messages[0].tool_calls).toHaveLength(1); - // Empty text → content is null (falsy) - expect(result.messages[0].content).toBeNull(); + // Empty text → content is "" (nullish coalescing preserves empty string) + expect(result.messages[0].content).toBe(""); }); }); @@ -1723,50 +1963,6 @@ describe("POST /model/{modelId}/invoke-with-response-stream (error fixture no ex // ─── Direct handler tests for req.method/req.url fallback branches ────────── -function createMockReq(overrides: Partial = {}): http.IncomingMessage { - return { - method: undefined, - url: undefined, - headers: {}, - ...overrides, - } as unknown as http.IncomingMessage; -} - -function createMockRes(): http.ServerResponse & { _written: string; _status: number } { - const res = { - _written: "", - _status: 0, - writableEnded: false, - statusCode: 0, - writeHead(status: number) { - res._status = status; - res.statusCode = status; - }, - setHeader() {}, - write(data: string) { - res._written += data; - return true; - }, - end(data?: string) { - if (data) res._written += data; - res.writableEnded = true; - }, - destroy() { - res.writableEnded = true; - }, - }; - return res as unknown as http.ServerResponse & { _written: string; _status: number }; -} - -function createDefaults(overrides: Partial = {}): HandlerDefaults { - return { - latency: 0, - chunkSize: 100, - logger: new Logger("silent"), - ...overrides, - }; -} - describe("handleConverse (direct handler call, method/url fallbacks)", () => { it("uses fallback for text response with undefined method/url", async () => { const fixture: Fixture = { diff --git a/src/__tests__/bedrock.test.ts b/src/__tests__/bedrock.test.ts index eb15c420..adb47c52 100644 --- a/src/__tests__/bedrock.test.ts +++ b/src/__tests__/bedrock.test.ts @@ -1,10 +1,11 @@ import { describe, it, expect, afterEach } from "vitest"; import * as http from "node:http"; -import type { Fixture, HandlerDefaults } from "../types.js"; +import type { Fixture } from "../types.js"; import { createServer, type ServerInstance } from "../server.js"; import { bedrockToCompletionRequest, handleBedrock, handleBedrockStream } from "../bedrock.js"; import { Journal } from "../journal.js"; import { Logger } from "../logger.js"; +import { createMockReq, createMockRes, createDefaults } from "./helpers/mock-res.js"; // --- helpers --- @@ -635,7 +636,7 @@ describe("bedrockToCompletionRequest (edge cases)", () => { }, "model", ); - expect(result.messages[0].tool_calls![0].id).toMatch(/^toolu_/); + expect(result.messages[0].tool_calls![0].id).toBe("tool_use_0"); }); it("handles assistant tool_use block with missing name (falls back to '')", () => { @@ -750,8 +751,8 @@ describe("bedrockToCompletionRequest (edge cases)", () => { }, "model", ); - // Empty array → no tool_use blocks, textContent is "" → null - expect(result.messages[0]).toEqual({ role: "assistant", content: null }); + // Empty array → no tool_use blocks, textContent is "" → preserved as "" (not coerced to null via ??) + expect(result.messages[0]).toEqual({ role: "assistant", content: "" }); }); it("handles user message with content blocks but no tool_results (text extraction)", () => { @@ -996,62 +997,6 @@ describe("POST /model/{modelId}/invoke (tool call with empty arguments)", () => // Direct handler tests for req.method/req.url fallback branches // --------------------------------------------------------------------------- -function createMockReq(overrides: Partial = {}): http.IncomingMessage { - return { - method: undefined, - url: undefined, - headers: {}, - ...overrides, - } as unknown as http.IncomingMessage; -} - -function createMockRes(): http.ServerResponse & { - _written: string; - _status: number; - _headers: Record; -} { - const res = { - _written: "", - _status: 0, - _headers: {} as Record, - writableEnded: false, - statusCode: 0, - writeHead(status: number, headers?: Record) { - res._status = status; - res.statusCode = status; - if (headers) Object.assign(res._headers, headers); - }, - setHeader(name: string, value: string) { - res._headers[name] = value; - }, - write(data: string) { - res._written += data; - return true; - }, - end(data?: string) { - if (data) res._written += data; - res.writableEnded = true; - }, - destroy() { - res.writableEnded = true; - }, - }; - return res as unknown as http.ServerResponse & { - _written: string; - _status: number; - _headers: Record; - }; -} - -function createDefaults(overrides: Partial = {}): HandlerDefaults { - return { - latency: 0, - chunkSize: 100, - logger: new Logger("silent"), - ...overrides, - }; -} - describe("handleBedrock (direct handler call, method/url fallbacks)", () => { it("uses fallback values when req.method and req.url are undefined", async () => { const fixture: Fixture = { @@ -1742,11 +1687,7 @@ describe("Bedrock webSearches warning", () => { "anthropic.claude-3-5-sonnet-20241022-v2:0", [fixture], journal, - { - latency: 0, - chunkSize: 100, - logger, - } as HandlerDefaults, + createDefaults({ logger }), () => {}, ); diff --git a/src/__tests__/helpers/mock-res.ts b/src/__tests__/helpers/mock-res.ts new file mode 100644 index 00000000..679f90a9 --- /dev/null +++ b/src/__tests__/helpers/mock-res.ts @@ -0,0 +1,59 @@ +import * as http from "node:http"; +import type { HandlerDefaults } from "../../types.js"; +import { Logger } from "../../logger.js"; + +export function createMockReq(overrides: Partial = {}): http.IncomingMessage { + return { + method: undefined, + url: undefined, + headers: {}, + ...overrides, + } as unknown as http.IncomingMessage; +} + +export function createMockRes(): http.ServerResponse & { + _written: string; + _status: number; + _headers: Record; +} { + const res = { + _written: "", + _status: 0, + _headers: {} as Record, + writableEnded: false, + statusCode: 0, + writeHead(status: number, headers?: Record) { + res._status = status; + res.statusCode = status; + if (headers) Object.assign(res._headers, headers); + }, + setHeader(name: string, value: string) { + res._headers[name] = value; + }, + write(data: string) { + res._written += data; + return true; + }, + end(data?: string) { + if (data) res._written += data; + res.writableEnded = true; + }, + destroy() { + res.writableEnded = true; + }, + }; + return res as unknown as http.ServerResponse & { + _written: string; + _status: number; + _headers: Record; + }; +} + +export function createDefaults(overrides: Partial = {}): HandlerDefaults { + return { + latency: 0, + chunkSize: 100, + logger: new Logger("silent"), + ...overrides, + }; +} diff --git a/src/__tests__/reasoning-all-providers.test.ts b/src/__tests__/reasoning-all-providers.test.ts index c26b238c..46fb1740 100644 --- a/src/__tests__/reasoning-all-providers.test.ts +++ b/src/__tests__/reasoning-all-providers.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; import * as http from "node:http"; import { crc32 } from "node:zlib"; import type { Fixture } from "../types.js"; @@ -7,7 +7,7 @@ import { buildBedrockStreamTextEvents } from "../bedrock.js"; // --- helpers --- -let instance: ServerInstance; +let instance: ServerInstance | null = null; let baseUrl: string; function post( @@ -176,15 +176,18 @@ const allFixtures: Fixture[] = [reasoningFixture, plainFixture]; // --- server lifecycle --- -beforeAll(async () => { +beforeEach(async () => { instance = await createServer(allFixtures); baseUrl = instance.url; }); -afterAll(async () => { - await new Promise((resolve) => { - instance.server.close(() => resolve()); - }); +afterEach(async () => { + if (instance) { + await new Promise((resolve) => { + instance!.server.close(() => resolve()); + }); + instance = null; + } }); // ─── OpenAI Chat Completions: Reasoning ───────────────────────────────────── @@ -549,7 +552,9 @@ describe("POST /model/{id}/converse-stream (reasoning streaming)", () => { .join(""); expect(fullThinking).toBe("Let me think step by step about this problem."); - expect(eventTypes[eventTypes.length - 1]).toBe("messageStop"); + // metadata event (with usage) follows messageStop in the Converse stream + expect(eventTypes[eventTypes.length - 2]).toBe("messageStop"); + expect(eventTypes[eventTypes.length - 1]).toBe("metadata"); }); it("no thinking block when reasoning is absent", async () => { diff --git a/src/bedrock-converse.ts b/src/bedrock-converse.ts index 4f7b78c5..6fb26bc6 100644 --- a/src/bedrock-converse.ts +++ b/src/bedrock-converse.ts @@ -75,6 +75,14 @@ function converseStopReason( return overrideFinishReason; } +/** + * Build Converse-format usage from fixture overrides. + * + * When no overrides are provided (the common case for mocks), all token + * counts default to zero. This is intentional — aimock is a mock server + * and does not perform real tokenisation. Callers that need non-zero + * usage should supply explicit `usage` overrides in their fixture. + */ function converseUsage(overrides?: ResponseOverrides): { inputTokens: number; outputTokens: number; @@ -113,7 +121,6 @@ function buildBedrockStreamTextEvents( events.push({ eventType: "contentBlockStart", payload: { - contentBlockIndex: blockIndex, contentBlockStart: { contentBlockIndex: blockIndex, start: { type: "thinking" } }, }, }); @@ -121,7 +128,6 @@ function buildBedrockStreamTextEvents( events.push({ eventType: "contentBlockDelta", payload: { - contentBlockIndex: blockIndex, contentBlockDelta: { contentBlockIndex: blockIndex, delta: { type: "thinking_delta", thinking: reasoning.slice(i, i + chunkSize) }, @@ -129,14 +135,16 @@ function buildBedrockStreamTextEvents( }, }); } - events.push({ eventType: "contentBlockStop", payload: { contentBlockIndex: blockIndex } }); + events.push({ + eventType: "contentBlockStop", + payload: { contentBlockStop: { contentBlockIndex: blockIndex } }, + }); } const textBlockIndex = reasoning ? 1 : 0; events.push({ eventType: "contentBlockStart", payload: { - contentBlockIndex: textBlockIndex, contentBlockStart: { contentBlockIndex: textBlockIndex, start: { type: "text" } }, }, }); @@ -144,7 +152,6 @@ function buildBedrockStreamTextEvents( events.push({ eventType: "contentBlockDelta", payload: { - contentBlockIndex: textBlockIndex, contentBlockDelta: { contentBlockIndex: textBlockIndex, delta: { type: "text_delta", text: content.slice(i, i + chunkSize) }, @@ -152,10 +159,20 @@ function buildBedrockStreamTextEvents( }, }); } - events.push({ eventType: "contentBlockStop", payload: { contentBlockIndex: textBlockIndex } }); + events.push({ + eventType: "contentBlockStop", + payload: { contentBlockStop: { contentBlockIndex: textBlockIndex } }, + }); events.push({ eventType: "messageStop", - payload: { stopReason: converseStopReason(overrides?.finishReason, "end_turn") }, + payload: { + messageStop: { stopReason: converseStopReason(overrides?.finishReason, "end_turn") }, + }, + }); + const usage = converseUsage(overrides); + events.push({ + eventType: "metadata", + payload: { metadata: { usage, metrics: { latencyMs: 0 } } }, }); return events; } @@ -168,11 +185,10 @@ function buildBedrockStreamContentWithToolCallsEvents( reasoning?: string, overrides?: ResponseOverrides, ): Array<{ eventType: string; payload: object }> { - const events = buildBedrockStreamTextEvents(content, chunkSize, reasoning, { - ...overrides, - finishReason: "stop", - }); - events.pop(); + const events = buildBedrockStreamTextEvents(content, chunkSize, reasoning, overrides); + // Remove trailing metadata + messageStop events — we re-emit them after tool blocks + events.pop(); // metadata + events.pop(); // messageStop let blockIndex = reasoning ? 2 : 1; for (const tc of toolCalls) { @@ -180,7 +196,6 @@ function buildBedrockStreamContentWithToolCallsEvents( events.push({ eventType: "contentBlockStart", payload: { - contentBlockIndex: blockIndex, contentBlockStart: { contentBlockIndex: blockIndex, start: { toolUse: { toolUseId, name: tc.name } }, @@ -192,7 +207,6 @@ function buildBedrockStreamContentWithToolCallsEvents( events.push({ eventType: "contentBlockDelta", payload: { - contentBlockIndex: blockIndex, contentBlockDelta: { contentBlockIndex: blockIndex, delta: { toolUse: { input: argsStr.slice(i, i + chunkSize) } }, @@ -200,12 +214,22 @@ function buildBedrockStreamContentWithToolCallsEvents( }, }); } - events.push({ eventType: "contentBlockStop", payload: { contentBlockIndex: blockIndex } }); + events.push({ + eventType: "contentBlockStop", + payload: { contentBlockStop: { contentBlockIndex: blockIndex } }, + }); blockIndex++; } events.push({ eventType: "messageStop", - payload: { stopReason: converseStopReason(overrides?.finishReason, "tool_use") }, + payload: { + messageStop: { stopReason: converseStopReason(overrides?.finishReason, "tool_use") }, + }, + }); + const usage = converseUsage(overrides); + events.push({ + eventType: "metadata", + payload: { metadata: { usage, metrics: { latencyMs: 0 } } }, }); return events; } @@ -226,7 +250,6 @@ function buildBedrockStreamToolCallEvents( events.push({ eventType: "contentBlockStart", payload: { - contentBlockIndex: tcIdx, contentBlockStart: { contentBlockIndex: tcIdx, start: { toolUse: { toolUseId, name: tc.name } }, @@ -238,7 +261,6 @@ function buildBedrockStreamToolCallEvents( events.push({ eventType: "contentBlockDelta", payload: { - contentBlockIndex: tcIdx, contentBlockDelta: { contentBlockIndex: tcIdx, delta: { toolUse: { input: argsStr.slice(i, i + chunkSize) } }, @@ -246,11 +268,21 @@ function buildBedrockStreamToolCallEvents( }, }); } - events.push({ eventType: "contentBlockStop", payload: { contentBlockIndex: tcIdx } }); + events.push({ + eventType: "contentBlockStop", + payload: { contentBlockStop: { contentBlockIndex: tcIdx } }, + }); } events.push({ eventType: "messageStop", - payload: { stopReason: converseStopReason(overrides?.finishReason, "tool_use") }, + payload: { + messageStop: { stopReason: converseStopReason(overrides?.finishReason, "tool_use") }, + }, + }); + const usage = converseUsage(overrides); + events.push({ + eventType: "metadata", + payload: { metadata: { usage, metrics: { latencyMs: 0 } } }, }); return events; } @@ -260,6 +292,7 @@ function buildBedrockStreamToolCallEvents( export function converseToCompletionRequest( req: ConverseRequest, modelId: string, + logger?: Logger, ): ChatCompletionRequest { const messages: ChatMessage[] = []; @@ -275,7 +308,17 @@ export function converseToCompletionRequest( if (msg.role === "user") { // Check for toolResult blocks const toolResults = msg.content.filter((b) => b.toolResult); - const textBlocks = msg.content.filter((b) => b.text !== undefined && !b.toolResult); + const textBlocks = msg.content.filter( + (b) => b.text !== undefined && b.text !== "" && !b.toolResult, + ); + const unsupportedBlocks = msg.content.filter( + (b) => b.text === undefined && !b.toolResult && !b.toolUse, + ); + if (unsupportedBlocks.length > 0 && logger) { + logger.warn( + `Converse user message contains unsupported content block types — these will be dropped during conversion`, + ); + } if (toolResults.length > 0) { for (const block of toolResults) { @@ -298,21 +341,21 @@ export function converseToCompletionRequest( // Plain user message const text = msg.content - .filter((b) => b.text !== undefined) + .filter((b) => b.text !== undefined && b.text !== "") .map((b) => b.text ?? "") .join(""); messages.push({ role: "user", content: text }); } else if (msg.role === "assistant") { const toolUseBlocks = msg.content.filter((b) => b.toolUse); const textContent = msg.content - .filter((b) => b.text !== undefined) + .filter((b) => b.text !== undefined && b.text !== "") .map((b) => b.text ?? "") .join(""); if (toolUseBlocks.length > 0) { messages.push({ role: "assistant", - content: textContent || null, + content: textContent ?? null, tool_calls: toolUseBlocks.map((b) => ({ id: b.toolUse!.toolUseId, type: "function" as const, @@ -323,7 +366,12 @@ export function converseToCompletionRequest( })), }); } else { - messages.push({ role: "assistant", content: textContent || null }); + messages.push({ role: "assistant", content: textContent ?? null }); + } + } else { + const warnMsg = `Unexpected message role "${msg.role}" in Converse request — skipping`; + if (logger) { + logger.warn(warnMsg); } } } @@ -336,7 +384,9 @@ export function converseToCompletionRequest( function: { name: t.toolSpec.name, description: t.toolSpec.description, - parameters: t.toolSpec.inputSchema, + parameters: (t.toolSpec.inputSchema && "json" in t.toolSpec.inputSchema + ? (t.toolSpec.inputSchema as Record).json + : t.toolSpec.inputSchema) as object | undefined, }, })); } @@ -518,7 +568,7 @@ export async function handleConverse( return; } - const completionReq = converseToCompletionRequest(converseReq, modelId); + const completionReq = converseToCompletionRequest(converseReq, modelId, logger); completionReq._endpointType = "chat"; const testId = getTestId(req); @@ -623,7 +673,7 @@ export async function handleConverse( const errBody = { type: "error", error: { - type: response.error.type || "invalid_request_error", + type: response.error.type ?? "invalid_request_error", message: response.error.message, }, }; @@ -681,6 +731,11 @@ export async function handleConverse( // Tool call response if (isToolCallResponse(response)) { + if ("webSearches" in response) { + logger.warn( + "webSearches in fixture response are not supported for Bedrock Converse API — ignoring", + ); + } const overrides = extractOverrides(response); journal.add({ method: req.method ?? "POST", @@ -775,7 +830,8 @@ export async function handleConverseStream( return; } - const completionReq = converseToCompletionRequest(converseReq, modelId); + const completionReq = converseToCompletionRequest(converseReq, modelId, logger); + completionReq.stream = true; completionReq._endpointType = "chat"; const testId = getTestId(req); @@ -882,7 +938,7 @@ export async function handleConverseStream( const errBody = { type: "error", error: { - type: response.error.type || "invalid_request_error", + type: response.error.type ?? "invalid_request_error", message: response.error.message, }, }; @@ -968,6 +1024,11 @@ export async function handleConverseStream( // Tool call response — stream as Event Stream if (isToolCallResponse(response)) { + if ("webSearches" in response) { + logger.warn( + "webSearches in fixture response are not supported for Bedrock Converse API — ignoring", + ); + } const overrides = extractOverrides(response); const journalEntry = journal.add({ method: req.method ?? "POST", diff --git a/src/bedrock.ts b/src/bedrock.ts index 31d9e213..f886ac0b 100644 --- a/src/bedrock.ts +++ b/src/bedrock.ts @@ -95,6 +95,15 @@ function bedrockStopReason( return overrideFinishReason; } +/** + * Build a Bedrock-style usage object from optional overrides. + * + * When no overrides are provided (the common case for mock fixtures), + * returns all-zero token counts. This is intentional — aimock does not + * attempt to estimate token usage from fixture content. Callers that + * need realistic usage numbers should set `usage` in their fixture's + * response overrides. + */ function bedrockUsage(overrides?: ResponseOverrides): { input_tokens: number; output_tokens: number; @@ -119,6 +128,7 @@ function extractTextContent(content: string | BedrockContentBlock[]): string { export function bedrockToCompletionRequest( req: BedrockRequest, modelId: string, + logger?: Logger, ): ChatCompletionRequest { const messages: ChatMessage[] = []; @@ -140,6 +150,17 @@ export function bedrockToCompletionRequest( if (msg.role === "user") { // Check for tool_result blocks if (typeof msg.content !== "string" && Array.isArray(msg.content)) { + // Warn about non-text content blocks that will be dropped (image, document, etc.) + const unsupportedBlocks = msg.content.filter( + (b) => b.type !== "text" && b.type !== "tool_result", + ); + if (unsupportedBlocks.length > 0 && logger) { + const types = [...new Set(unsupportedBlocks.map((b) => b.type))].join(", "); + logger.warn( + `Bedrock user message contains unsupported content block types [${types}] — these will be dropped during conversion`, + ); + } + const toolResults = msg.content.filter((b) => b.type === "tool_result"); const textBlocks = msg.content.filter((b) => b.type === "text"); @@ -183,22 +204,35 @@ export function bedrockToCompletionRequest( if (toolUseBlocks.length > 0) { messages.push({ role: "assistant", - content: textContent || null, - tool_calls: toolUseBlocks.map((b) => ({ - id: b.id ?? generateToolUseId(), - type: "function" as const, - function: { - name: b.name ?? "", - arguments: typeof b.input === "string" ? b.input : JSON.stringify(b.input ?? {}), - }, - })), + content: textContent ?? null, + tool_calls: toolUseBlocks.map((b, index) => { + if (!b.id && logger) { + logger.warn( + `Bedrock assistant tool_use block at index ${index} is missing an id — using deterministic fallback "tool_use_${index}"`, + ); + } + return { + id: b.id ?? `tool_use_${index}`, + type: "function" as const, + function: { + name: b.name ?? "", + arguments: typeof b.input === "string" ? b.input : JSON.stringify(b.input ?? {}), + }, + }; + }), }); } else { - messages.push({ role: "assistant", content: textContent || null }); + messages.push({ role: "assistant", content: textContent ?? null }); } } else { messages.push({ role: "assistant", content: null }); } + } else { + if (logger) { + logger.warn( + `Bedrock message has unexpected role "${(msg as { role: string }).role}" — skipping`, + ); + } } } @@ -347,7 +381,7 @@ export async function handleBedrock( } // Convert to ChatCompletionRequest for fixture matching - const completionReq = bedrockToCompletionRequest(bedrockReq, modelId); + const completionReq = bedrockToCompletionRequest(bedrockReq, modelId, logger); completionReq._endpointType = "chat"; const testId = getTestId(req); @@ -449,7 +483,8 @@ export async function handleBedrock( body: completionReq, response: { status, fixture }, }); - // Anthropic-style error format (Bedrock uses Claude): { type: "error", error: { type, message } } + // Bedrock Claude error format: { type: "error", error: { type, message } } + // Uses ?? (nullish coalescing) intentionally — preserves explicit empty-string types from fixtures. const anthropicError = { type: "error", error: { @@ -526,6 +561,9 @@ export async function handleBedrock( // Tool call response if (isToolCallResponse(response)) { + if ("webSearches" in response) { + logger.warn("webSearches in fixture response are not supported for Bedrock API — ignoring"); + } const overrides = extractOverrides(response); journal.add({ method: req.method ?? "POST", @@ -938,7 +976,8 @@ export async function handleBedrockStream( return; } - const completionReq = bedrockToCompletionRequest(bedrockReq, modelId); + const completionReq = bedrockToCompletionRequest(bedrockReq, modelId, logger); + completionReq.stream = true; completionReq._endpointType = "chat"; const testId = getTestId(req); @@ -1042,7 +1081,8 @@ export async function handleBedrockStream( body: completionReq, response: { status, fixture }, }); - // Anthropic-style error format (Bedrock uses Claude): { type: "error", error: { type, message } } + // Bedrock Claude error format: { type: "error", error: { type, message } } + // Uses ?? (nullish coalescing) intentionally — preserves explicit empty-string types from fixtures. const anthropicError = { type: "error", error: { @@ -1130,6 +1170,9 @@ export async function handleBedrockStream( // Tool call response — stream as Event Stream if (isToolCallResponse(response)) { + if ("webSearches" in response) { + logger.warn("webSearches in fixture response are not supported for Bedrock API — ignoring"); + } const overrides = extractOverrides(response); const journalEntry = journal.add({ method: req.method ?? "POST",