From d362f635245234dade51f0240a84e3e2f0e9bc44 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Tue, 28 Apr 2026 18:02:03 -0700 Subject: [PATCH 1/4] feat: add turnIndex and hasToolResult stateless match criteria turnIndex counts assistant messages in the request conversation history. hasToolResult checks for the presence of tool-role messages. Both are stateless and safe for shared aimock instances with concurrent clients. Includes onTurn() convenience method and unit tests. --- src/__tests__/router.test.ts | 166 +++++++++++++++++++++++++++++++++++ src/fixture-loader.ts | 41 ++++++++- src/journal.ts | 4 +- src/llmock.ts | 9 ++ src/router.ts | 10 +++ src/types.ts | 4 + 6 files changed, 229 insertions(+), 5 deletions(-) diff --git a/src/__tests__/router.test.ts b/src/__tests__/router.test.ts index 40c20043..fd6a7555 100644 --- a/src/__tests__/router.test.ts +++ b/src/__tests__/router.test.ts @@ -582,6 +582,172 @@ describe("matchFixture — sequenceIndex", () => { }); }); +// --------------------------------------------------------------------------- +// matchFixture — turnIndex +// --------------------------------------------------------------------------- + +describe("matchFixture — turnIndex", () => { + it("matches when assistant message count equals turnIndex", () => { + const fixture = makeFixture({ userMessage: "hello", turnIndex: 1 }); + const req = makeReq({ + messages: [ + { role: "system", content: "you are helpful" }, + { role: "user", content: "hello" }, + { role: "assistant", content: "hi there" }, + { role: "user", content: "hello" }, + ], + }); + expect(matchFixture([fixture], req)).toBe(fixture); + }); + + it("skips when assistant message count does not equal turnIndex", () => { + const fixture = makeFixture({ userMessage: "hello", turnIndex: 2 }); + const req = makeReq({ + messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "hi there" }, + { role: "user", content: "hello" }, + ], + }); + expect(matchFixture([fixture], req)).toBeNull(); + }); + + it("turnIndex 0 matches when no assistant messages present", () => { + const fixture = makeFixture({ userMessage: "hello", turnIndex: 0 }); + const req = makeReq({ + messages: [ + { role: "system", content: "you are helpful" }, + { role: "user", content: "hello" }, + ], + }); + expect(matchFixture([fixture], req)).toBe(fixture); + }); + + it("selects correct fixture from turnIndex sequence", () => { + const turn0 = makeFixture({ userMessage: "hello", turnIndex: 0 }, { content: "turn-0" }); + const turn1 = makeFixture({ userMessage: "hello", turnIndex: 1 }, { content: "turn-1" }); + const turn2 = makeFixture({ userMessage: "hello", turnIndex: 2 }, { content: "turn-2" }); + + const req0 = makeReq({ + messages: [{ role: "user", content: "hello" }], + }); + expect(matchFixture([turn0, turn1, turn2], req0)).toBe(turn0); + + const req1 = makeReq({ + messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "reply" }, + { role: "user", content: "hello" }, + ], + }); + expect(matchFixture([turn0, turn1, turn2], req1)).toBe(turn1); + + const req2 = makeReq({ + messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "reply1" }, + { role: "user", content: "hello" }, + { role: "assistant", content: "reply2" }, + { role: "user", content: "hello" }, + ], + }); + expect(matchFixture([turn0, turn1, turn2], req2)).toBe(turn2); + }); + + it("falls through to non-turnIndex fixture when no turnIndex matches", () => { + const turnOnly = makeFixture({ userMessage: "hello", turnIndex: 0 }, { content: "turn-0" }); + const fallback = makeFixture({ userMessage: "hello" }, { content: "fallback" }); + const req = makeReq({ + messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "reply1" }, + { role: "user", content: "hello" }, + { role: "assistant", content: "reply2" }, + { role: "user", content: "hello" }, + ], + }); + expect(matchFixture([turnOnly, fallback], req)).toBe(fallback); + }); +}); + +// --------------------------------------------------------------------------- +// matchFixture — hasToolResult +// --------------------------------------------------------------------------- + +describe("matchFixture — hasToolResult", () => { + it("matches hasToolResult: true when tool messages present", () => { + const fixture = makeFixture({ userMessage: "hello", hasToolResult: true }); + const req = makeReq({ + messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "calling tool" }, + { role: "tool", content: "tool output" }, + { role: "user", content: "hello" }, + ], + }); + expect(matchFixture([fixture], req)).toBe(fixture); + }); + + it("skips hasToolResult: true when no tool messages present", () => { + const fixture = makeFixture({ userMessage: "hello", hasToolResult: true }); + const req = makeReq({ + messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "reply" }, + { role: "user", content: "hello" }, + ], + }); + expect(matchFixture([fixture], req)).toBeNull(); + }); + + it("matches hasToolResult: false when no tool messages present", () => { + const fixture = makeFixture({ userMessage: "hello", hasToolResult: false }); + const req = makeReq({ + messages: [{ role: "user", content: "hello" }], + }); + expect(matchFixture([fixture], req)).toBe(fixture); + }); + + it("skips hasToolResult: false when tool messages present", () => { + const fixture = makeFixture({ userMessage: "hello", hasToolResult: false }); + const req = makeReq({ + messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "calling tool" }, + { role: "tool", content: "tool output" }, + { role: "user", content: "hello" }, + ], + }); + expect(matchFixture([fixture], req)).toBeNull(); + }); + + it("discriminates 2-step HITL flow with hasToolResult", () => { + const beforeTool = makeFixture( + { userMessage: "hello", hasToolResult: false }, + { content: "before-tool" }, + ); + const afterTool = makeFixture( + { userMessage: "hello", hasToolResult: true }, + { content: "after-tool" }, + ); + + const reqBefore = makeReq({ + messages: [{ role: "user", content: "hello" }], + }); + expect(matchFixture([beforeTool, afterTool], reqBefore)).toBe(beforeTool); + + const reqAfter = makeReq({ + messages: [ + { role: "user", content: "hello" }, + { role: "assistant", content: "calling tool" }, + { role: "tool", content: "result" }, + { role: "user", content: "hello" }, + ], + }); + expect(matchFixture([beforeTool, afterTool], reqAfter)).toBe(afterTool); + }); +}); + // --------------------------------------------------------------------------- // matchFixture — first-match-wins // --------------------------------------------------------------------------- diff --git a/src/fixture-loader.ts b/src/fixture-loader.ts index 40d2ddbb..0d7a2100 100644 --- a/src/fixture-loader.ts +++ b/src/fixture-loader.ts @@ -59,6 +59,12 @@ export function entryToFixture(entry: FixtureFileEntry): Fixture { responseFormat: entry.match.responseFormat, endpoint: entry.match.endpoint, ...(entry.match.sequenceIndex !== undefined && { sequenceIndex: entry.match.sequenceIndex }), + ...(entry.match.turnIndex !== undefined && { + turnIndex: entry.match.turnIndex, + }), + ...(entry.match.hasToolResult !== undefined && { + hasToolResult: entry.match.hasToolResult, + }), }, response: normalizeResponse(entry.response), ...(entry.latency !== undefined && { latency: entry.latency }), @@ -525,12 +531,37 @@ export function validateFixtures(fixtures: Fixture[]): ValidationResult[] { } } + // Match field type checks + if (f.match.turnIndex !== undefined) { + if ( + typeof f.match.turnIndex !== "number" || + f.match.turnIndex < 0 || + !Number.isInteger(f.match.turnIndex) + ) { + results.push({ + severity: "error", + fixtureIndex: i, + message: "match.turnIndex must be a non-negative integer", + }); + } + } + if (f.match.hasToolResult !== undefined && typeof f.match.hasToolResult !== "boolean") { + results.push({ + severity: "error", + fixtureIndex: i, + message: `match.hasToolResult must be a boolean, got ${typeof f.match.hasToolResult}`, + }); + } + // --- Warning checks --- - // Duplicate userMessage shadowing + // Duplicate userMessage shadowing — include turnIndex, hasToolResult, and + // sequenceIndex in the dedup key so that fixtures which share a userMessage + // but differ on those fields are NOT considered duplicates. const um = f.match.userMessage; if (typeof um === "string" && um) { - const prev = seenUserMessages.get(um); + const dedupKey = `${um}|${f.match.turnIndex}|${f.match.hasToolResult}|${f.match.sequenceIndex}`; + const prev = seenUserMessages.get(dedupKey); if (prev !== undefined) { results.push({ severity: "warning", @@ -538,7 +569,7 @@ export function validateFixtures(fixtures: Fixture[]): ValidationResult[] { message: `duplicate userMessage '${um}' — shadows fixture ${prev}`, }); } else { - seenUserMessages.set(um, i); + seenUserMessages.set(dedupKey, i); } } @@ -552,7 +583,9 @@ export function validateFixtures(fixtures: Fixture[]): ValidationResult[] { match.toolCallId !== undefined || match.toolName !== undefined || match.model !== undefined || - match.predicate !== undefined; + match.predicate !== undefined || + match.turnIndex !== undefined || + match.hasToolResult !== undefined; if (!hasDiscriminator && i < fixtures.length - 1) { results.push({ diff --git a/src/journal.ts b/src/journal.ts index b00fbd62..9a0d8f72 100644 --- a/src/journal.ts +++ b/src/journal.ts @@ -26,7 +26,9 @@ function matchCriteriaEqual(a: FixtureMatch, b: FixtureMatch): boolean { fieldEqual(a.model, b.model) && fieldEqual(a.responseFormat, b.responseFormat) && fieldEqual(a.predicate, b.predicate) && - fieldEqual(a.endpoint, b.endpoint) + fieldEqual(a.endpoint, b.endpoint) && + fieldEqual(a.turnIndex, b.turnIndex) && + fieldEqual(a.hasToolResult, b.hasToolResult) ); } diff --git a/src/llmock.ts b/src/llmock.ts index a5c8dac1..519dabda 100644 --- a/src/llmock.ts +++ b/src/llmock.ts @@ -130,6 +130,15 @@ export class LLMock { return this.on({ toolCallId: id }, response, opts); } + onTurn( + turn: number, + pattern: string | RegExp, + response: FixtureFileResponse, + opts?: FixtureOpts, + ): this { + return this.on({ userMessage: pattern, turnIndex: turn }, response, opts); + } + onImage(prompt: string | RegExp, response: ImageResponse): this { return this.addFixture({ match: { userMessage: prompt, endpoint: "image" }, diff --git a/src/router.ts b/src/router.ts index 65f6be8d..df30c146 100644 --- a/src/router.ts +++ b/src/router.ts @@ -140,6 +140,16 @@ export function matchFixture( if (count !== match.sequenceIndex) continue; } + if (match.turnIndex !== undefined) { + const assistantCount = effective.messages.filter((m) => m.role === "assistant").length; + if (assistantCount !== match.turnIndex) continue; + } + + if (match.hasToolResult !== undefined) { + const hasTool = effective.messages.some((m) => m.role === "tool"); + if (hasTool !== match.hasToolResult) continue; + } + return fixture; } diff --git a/src/types.ts b/src/types.ts index e434f37d..9d8a07c1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -72,6 +72,8 @@ export interface FixtureMatch { predicate?: (req: ChatCompletionRequest) => boolean; /** Which occurrence of this match to respond to (0-indexed). Undefined means match any. */ sequenceIndex?: number; + turnIndex?: number; + hasToolResult?: boolean; endpoint?: "chat" | "image" | "speech" | "transcription" | "video" | "embedding"; } @@ -277,6 +279,8 @@ export interface FixtureFileEntry { model?: string; responseFormat?: string; sequenceIndex?: number; + turnIndex?: number; + hasToolResult?: boolean; endpoint?: "chat" | "image" | "speech" | "transcription" | "video" | "embedding"; // predicate not supported in JSON files }; From cc8f0dd1a41c3d272a07fa5ed57579eff5122b5f Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Tue, 28 Apr 2026 18:02:08 -0700 Subject: [PATCH 2/4] test: integration tests for turnIndex and hasToolResult Cross-provider tests (OpenAI, Anthropic, Gemini), concurrency verification, combined AND-logic, JSON fixture loading, and validateFixtures type/shadowing coverage. --- src/__tests__/fixture-loader.test.ts | 107 +++++ src/__tests__/turn-index.test.ts | 653 +++++++++++++++++++++++++++ 2 files changed, 760 insertions(+) create mode 100644 src/__tests__/turn-index.test.ts diff --git a/src/__tests__/fixture-loader.test.ts b/src/__tests__/fixture-loader.test.ts index ee82303e..9a909e26 100644 --- a/src/__tests__/fixture-loader.test.ts +++ b/src/__tests__/fixture-loader.test.ts @@ -800,6 +800,66 @@ describe("validateFixtures", () => { expect(validateFixtures(fixtures)).toHaveLength(0); }); + // --- match.turnIndex / match.hasToolResult type checks --- + + it("error: turnIndex is negative", () => { + const fixtures = [makeFixture({ match: { userMessage: "test", turnIndex: -1 } })]; + const results = validateFixtures(fixtures); + expect(results.some((r) => r.severity === "error" && r.message.includes("turnIndex"))).toBe( + true, + ); + }); + + it("error: turnIndex is a float", () => { + const fixtures = [makeFixture({ match: { userMessage: "test", turnIndex: 1.5 } })]; + const results = validateFixtures(fixtures); + expect(results.some((r) => r.severity === "error" && r.message.includes("turnIndex"))).toBe( + true, + ); + }); + + it("error: turnIndex is a string", () => { + const fixtures = [makeFixture({ match: { userMessage: "test", turnIndex: "zero" as never } })]; + const results = validateFixtures(fixtures); + expect(results.some((r) => r.severity === "error" && r.message.includes("turnIndex"))).toBe( + true, + ); + }); + + it("no error: turnIndex is 0 (falsy but valid)", () => { + const fixtures = [makeFixture({ match: { userMessage: "test", turnIndex: 0 } })]; + const results = validateFixtures(fixtures); + expect(results.filter((r) => r.message.includes("turnIndex"))).toHaveLength(0); + }); + + it("no error: turnIndex is a positive integer", () => { + const fixtures = [makeFixture({ match: { userMessage: "test", turnIndex: 3 } })]; + const results = validateFixtures(fixtures); + expect(results.filter((r) => r.message.includes("turnIndex"))).toHaveLength(0); + }); + + it("error: hasToolResult is a string", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "test", hasToolResult: "yes" as never } }), + ]; + const results = validateFixtures(fixtures); + expect(results.some((r) => r.severity === "error" && r.message.includes("hasToolResult"))).toBe( + true, + ); + }); + + it("no error: hasToolResult is false (falsy but valid)", () => { + const fixtures = [makeFixture({ match: { userMessage: "test", hasToolResult: false } })]; + const results = validateFixtures(fixtures); + expect(results.filter((r) => r.message.includes("hasToolResult"))).toHaveLength(0); + }); + + it("no error: hasToolResult is true", () => { + const fixtures = [makeFixture({ match: { userMessage: "test", hasToolResult: true } })]; + const results = validateFixtures(fixtures); + expect(results.filter((r) => r.message.includes("hasToolResult"))).toHaveLength(0); + }); + // --- Warning checks --- it("warning: duplicate userMessage", () => { @@ -813,6 +873,53 @@ describe("validateFixtures", () => { ); }); + it("no warning: same userMessage but different turnIndex", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "hello", turnIndex: 0 } }), + makeFixture({ match: { userMessage: "hello", turnIndex: 1 } }), + ]; + const results = validateFixtures(fixtures); + const duplicateWarnings = results.filter( + (r) => r.severity === "warning" && r.message.includes("duplicate"), + ); + expect(duplicateWarnings).toHaveLength(0); + }); + + it("no warning: same userMessage but different hasToolResult", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "hello", hasToolResult: false } }), + makeFixture({ match: { userMessage: "hello", hasToolResult: true } }), + ]; + const results = validateFixtures(fixtures); + const duplicateWarnings = results.filter( + (r) => r.severity === "warning" && r.message.includes("duplicate"), + ); + expect(duplicateWarnings).toHaveLength(0); + }); + + it("no warning: same userMessage but different sequenceIndex", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "hello", sequenceIndex: 0 } }), + makeFixture({ match: { userMessage: "hello", sequenceIndex: 1 } }), + ]; + const results = validateFixtures(fixtures); + const duplicateWarnings = results.filter( + (r) => r.severity === "warning" && r.message.includes("duplicate"), + ); + expect(duplicateWarnings).toHaveLength(0); + }); + + it("warning: same userMessage with identical turnIndex/hasToolResult/sequenceIndex", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "hello", turnIndex: 1, hasToolResult: true } }), + makeFixture({ match: { userMessage: "hello", turnIndex: 1, hasToolResult: true } }), + ]; + const results = validateFixtures(fixtures); + expect(results.some((r) => r.severity === "warning" && r.message.includes("duplicate"))).toBe( + true, + ); + }); + it("warning: catch-all not in last position", () => { const fixtures = [makeFixture({ match: {} }), makeFixture({ match: { userMessage: "hello" } })]; const results = validateFixtures(fixtures); diff --git a/src/__tests__/turn-index.test.ts b/src/__tests__/turn-index.test.ts new file mode 100644 index 00000000..6a32cb26 --- /dev/null +++ b/src/__tests__/turn-index.test.ts @@ -0,0 +1,653 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { LLMock } from "../llmock.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function chatPost( + baseUrl: string, + messages: Array>, + extra?: Record, +) { + return fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: "gpt-4", messages, stream: false, ...extra }), + }); +} + +function claudePost( + baseUrl: string, + messages: Array>, + extra?: Record, +) { + return fetch(`${baseUrl}/v1/messages`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-api-key": "test-key", + "anthropic-version": "2023-06-01", + }, + body: JSON.stringify({ + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + messages, + stream: false, + ...extra, + }), + }); +} + +function geminiPost( + baseUrl: string, + contents: Array>, + extra?: Record, +) { + return fetch(`${baseUrl}/v1beta/models/gemini-2.0-flash:generateContent?key=test-key`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ contents, ...extra }), + }); +} + +// --------------------------------------------------------------------------- +// 1. turnIndex integration tests +// --------------------------------------------------------------------------- + +describe("turnIndex — OpenAI Chat 2-step HITL flow", () => { + let mock: LLMock; + + beforeAll(async () => { + mock = new LLMock(); + await mock.start(); + }); + + afterAll(async () => { + await mock.stop(); + }); + + it("step 0: no assistant messages → turnIndex 0 matches tool call", async () => { + mock.reset(); + mock.on( + { userMessage: "plan a trip to mars", turnIndex: 0 }, + { + toolCalls: [{ name: "plan_trip", arguments: '{"destination":"mars"}', id: "call_plan_1" }], + }, + ); + mock.on( + { userMessage: "plan a trip to mars", turnIndex: 1 }, + { content: "Your trip to Mars is booked!" }, + ); + + // Step 0: just user message, no assistant messages → turnIndex 0 + const res0 = await chatPost(mock.url, [{ role: "user", content: "plan a trip to mars" }]); + expect(res0.status).toBe(200); + const body0 = (await res0.json()) as { + choices: { + message: { tool_calls: { function: { name: string } }[] }; + finish_reason: string; + }[]; + }; + expect(body0.choices[0].message.tool_calls[0].function.name).toBe("plan_trip"); + expect(body0.choices[0].finish_reason).toBe("tool_calls"); + }); + + it("step 1: one assistant message → turnIndex 1 matches text follow-up", async () => { + mock.reset(); + mock.on( + { userMessage: "plan a trip to mars", turnIndex: 0 }, + { + toolCalls: [{ name: "plan_trip", arguments: '{"destination":"mars"}', id: "call_plan_1" }], + }, + ); + mock.on( + { userMessage: "plan a trip to mars", turnIndex: 1 }, + { content: "Your trip to Mars is booked!" }, + ); + + // Step 1: user + assistant (tool_calls) + tool result → turnIndex 1 + const res1 = await chatPost(mock.url, [ + { role: "user", content: "plan a trip to mars" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_plan_1", + type: "function", + function: { name: "plan_trip", arguments: '{"destination":"mars"}' }, + }, + ], + }, + { role: "tool", tool_call_id: "call_plan_1", content: '{"status":"confirmed"}' }, + ]); + expect(res1.status).toBe(200); + const body1 = (await res1.json()) as { choices: { message: { content: string } }[] }; + expect(body1.choices[0].message.content).toBe("Your trip to Mars is booked!"); + }); +}); + +describe("turnIndex — OpenAI Chat 4-step subagent flow", () => { + let mock: LLMock; + + beforeAll(async () => { + mock = new LLMock(); + await mock.start(); + }); + + afterAll(async () => { + await mock.stop(); + }); + + it("4 turns return distinct content based on turnIndex", async () => { + mock.reset(); + mock.on({ userMessage: "subagent", turnIndex: 0 }, { content: "turn-0-response" }); + mock.on({ userMessage: "subagent", turnIndex: 1 }, { content: "turn-1-response" }); + mock.on({ userMessage: "subagent", turnIndex: 2 }, { content: "turn-2-response" }); + mock.on({ userMessage: "subagent", turnIndex: 3 }, { content: "turn-3-response" }); + + const responses: string[] = []; + + // Turn 0: just user message + const res0 = await chatPost(mock.url, [{ role: "user", content: "subagent" }]); + responses.push( + ((await res0.json()) as { choices: { message: { content: string } }[] }).choices[0].message + .content, + ); + + // Turn 1: user + 1 assistant + const res1 = await chatPost(mock.url, [ + { role: "user", content: "subagent" }, + { role: "assistant", content: "turn-0-response" }, + ]); + responses.push( + ((await res1.json()) as { choices: { message: { content: string } }[] }).choices[0].message + .content, + ); + + // Turn 2: user + 2 assistants + const res2 = await chatPost(mock.url, [ + { role: "user", content: "subagent" }, + { role: "assistant", content: "turn-0-response" }, + { role: "assistant", content: "turn-1-response" }, + ]); + responses.push( + ((await res2.json()) as { choices: { message: { content: string } }[] }).choices[0].message + .content, + ); + + // Turn 3: user + 3 assistants + const res3 = await chatPost(mock.url, [ + { role: "user", content: "subagent" }, + { role: "assistant", content: "turn-0-response" }, + { role: "assistant", content: "turn-1-response" }, + { role: "assistant", content: "turn-2-response" }, + ]); + responses.push( + ((await res3.json()) as { choices: { message: { content: string } }[] }).choices[0].message + .content, + ); + + expect(responses).toEqual([ + "turn-0-response", + "turn-1-response", + "turn-2-response", + "turn-3-response", + ]); + }); +}); + +describe("turnIndex — concurrency (stateless verification)", () => { + let mock: LLMock; + + beforeAll(async () => { + mock = new LLMock(); + await mock.start(); + }); + + afterAll(async () => { + await mock.stop(); + }); + + it("5 concurrent step-0 and 5 concurrent step-1 all get correct responses", async () => { + mock.reset(); + mock.on({ userMessage: "concurrent-turn", turnIndex: 0 }, { content: "first-turn" }); + mock.on({ userMessage: "concurrent-turn", turnIndex: 1 }, { content: "second-turn" }); + + // Step 0 messages: no assistant messages + const step0Messages = [{ role: "user", content: "concurrent-turn" }]; + // Step 1 messages: one assistant message + const step1Messages = [ + { role: "user", content: "concurrent-turn" }, + { role: "assistant", content: "first-turn" }, + ]; + + // Fire 5 step-0 and 5 step-1 concurrently + const allPromises = [ + ...Array.from({ length: 5 }, () => chatPost(mock.url, step0Messages)), + ...Array.from({ length: 5 }, () => chatPost(mock.url, step1Messages)), + ]; + + const results = await Promise.all(allPromises); + + const step0Results: string[] = []; + const step1Results: string[] = []; + + for (let i = 0; i < 10; i++) { + expect(results[i].status).toBe(200); + const body = (await results[i].json()) as { + choices: { message: { content: string } }[]; + }; + if (i < 5) { + step0Results.push(body.choices[0].message.content); + } else { + step1Results.push(body.choices[0].message.content); + } + } + + // ALL step-0 requests get "first-turn" (no counter drift) + expect(step0Results).toEqual(Array(5).fill("first-turn")); + // ALL step-1 requests get "second-turn" + expect(step1Results).toEqual(Array(5).fill("second-turn")); + }); +}); + +describe("turnIndex — Anthropic Claude cross-provider", () => { + let mock: LLMock; + + beforeAll(async () => { + mock = new LLMock(); + await mock.start(); + }); + + afterAll(async () => { + await mock.stop(); + }); + + it("turnIndex 0 and 1 work through Claude message normalization", async () => { + mock.reset(); + mock.on({ userMessage: "claude-turn", turnIndex: 0 }, { content: "claude-first" }); + mock.on({ userMessage: "claude-turn", turnIndex: 1 }, { content: "claude-second" }); + + // Turn 0: just user message → no assistant messages → turnIndex 0 + const res0 = await claudePost(mock.url, [{ role: "user", content: "claude-turn" }]); + expect(res0.status).toBe(200); + const body0 = (await res0.json()) as { content: { type: string; text: string }[] }; + expect(body0.content[0].text).toBe("claude-first"); + + // Turn 1: user + assistant → 1 assistant message → turnIndex 1 + const res1 = await claudePost(mock.url, [ + { role: "user", content: "claude-turn" }, + { role: "assistant", content: "claude-first" }, + { role: "user", content: "claude-turn" }, + ]); + expect(res1.status).toBe(200); + const body1 = (await res1.json()) as { content: { type: string; text: string }[] }; + expect(body1.content[0].text).toBe("claude-second"); + }); +}); + +describe("turnIndex — Gemini cross-provider", () => { + let mock: LLMock; + + beforeAll(async () => { + mock = new LLMock(); + await mock.start(); + }); + + afterAll(async () => { + await mock.stop(); + }); + + it("turnIndex 0 and 1 work through Gemini content normalization", async () => { + mock.reset(); + mock.on({ userMessage: "gemini-turn", turnIndex: 0 }, { content: "gemini-first" }); + mock.on({ userMessage: "gemini-turn", turnIndex: 1 }, { content: "gemini-second" }); + + // Turn 0: just user content → no model (assistant) contents → turnIndex 0 + const res0 = await geminiPost(mock.url, [{ role: "user", parts: [{ text: "gemini-turn" }] }]); + expect(res0.status).toBe(200); + type GeminiBody = { + candidates: { content: { parts: { text: string }[] } }[]; + }; + const body0 = (await res0.json()) as GeminiBody; + expect(body0.candidates[0].content.parts[0].text).toBe("gemini-first"); + + // Turn 1: user + model (assistant equivalent) → turnIndex 1 + const res1 = await geminiPost(mock.url, [ + { role: "user", parts: [{ text: "gemini-turn" }] }, + { role: "model", parts: [{ text: "gemini-first" }] }, + { role: "user", parts: [{ text: "gemini-turn" }] }, + ]); + expect(res1.status).toBe(200); + const body1 = (await res1.json()) as GeminiBody; + expect(body1.candidates[0].content.parts[0].text).toBe("gemini-second"); + }); +}); + +// --------------------------------------------------------------------------- +// 2. hasToolResult integration tests +// --------------------------------------------------------------------------- + +describe("hasToolResult — HITL 2-step discrimination", () => { + let mock: LLMock; + + beforeAll(async () => { + mock = new LLMock(); + await mock.start(); + }); + + afterAll(async () => { + await mock.stop(); + }); + + it("hasToolResult:false matches when no tool messages; hasToolResult:true matches with tool messages", async () => { + mock.reset(); + mock.on( + { userMessage: "weather", hasToolResult: false }, + { + toolCalls: [{ name: "get_weather", arguments: '{"city":"NYC"}', id: "call_w1" }], + }, + ); + mock.on( + { userMessage: "weather", hasToolResult: true }, + { content: "The weather in NYC is sunny, 72F." }, + ); + + // First request: no tool messages → hasToolResult false → tool call + const res0 = await chatPost(mock.url, [{ role: "user", content: "weather" }]); + expect(res0.status).toBe(200); + const body0 = (await res0.json()) as { + choices: { + message: { tool_calls: { function: { name: string } }[] }; + finish_reason: string; + }[]; + }; + expect(body0.choices[0].message.tool_calls[0].function.name).toBe("get_weather"); + expect(body0.choices[0].finish_reason).toBe("tool_calls"); + + // Second request: includes tool result → hasToolResult true → text + const res1 = await chatPost(mock.url, [ + { role: "user", content: "weather" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_w1", + type: "function", + function: { name: "get_weather", arguments: '{"city":"NYC"}' }, + }, + ], + }, + { role: "tool", tool_call_id: "call_w1", content: '{"temp":72,"condition":"sunny"}' }, + ]); + expect(res1.status).toBe(200); + const body1 = (await res1.json()) as { choices: { message: { content: string } }[] }; + expect(body1.choices[0].message.content).toBe("The weather in NYC is sunny, 72F."); + }); +}); + +describe("hasToolResult — concurrency", () => { + let mock: LLMock; + + beforeAll(async () => { + mock = new LLMock(); + await mock.start(); + }); + + afterAll(async () => { + await mock.stop(); + }); + + it("5 no-tool + 5 with-tool concurrent requests all discriminate correctly", async () => { + mock.reset(); + mock.on( + { userMessage: "concurrent-tool", hasToolResult: false }, + { content: "no-tool-response" }, + ); + mock.on( + { userMessage: "concurrent-tool", hasToolResult: true }, + { content: "has-tool-response" }, + ); + + const noToolMessages = [{ role: "user", content: "concurrent-tool" }]; + const withToolMessages = [ + { role: "user", content: "concurrent-tool" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_ct1", + type: "function", + function: { name: "some_tool", arguments: "{}" }, + }, + ], + }, + { role: "tool", tool_call_id: "call_ct1", content: "result" }, + ]; + + const allPromises = [ + ...Array.from({ length: 5 }, () => chatPost(mock.url, noToolMessages)), + ...Array.from({ length: 5 }, () => chatPost(mock.url, withToolMessages)), + ]; + + const results = await Promise.all(allPromises); + + const noToolResults: string[] = []; + const withToolResults: string[] = []; + + for (let i = 0; i < 10; i++) { + expect(results[i].status).toBe(200); + const body = (await results[i].json()) as { + choices: { message: { content: string } }[]; + }; + if (i < 5) { + noToolResults.push(body.choices[0].message.content); + } else { + withToolResults.push(body.choices[0].message.content); + } + } + + expect(noToolResults).toEqual(Array(5).fill("no-tool-response")); + expect(withToolResults).toEqual(Array(5).fill("has-tool-response")); + }); +}); + +// --------------------------------------------------------------------------- +// 3. Combined turnIndex + hasToolResult (AND logic) +// --------------------------------------------------------------------------- + +describe("combined turnIndex + hasToolResult — AND logic", () => { + let mock: LLMock; + + beforeAll(async () => { + mock = new LLMock(); + await mock.start(); + }); + + afterAll(async () => { + await mock.stop(); + }); + + it("both conditions must hold for the specific fixture to match", async () => { + mock.reset(); + // Specific: turnIndex 1 AND hasToolResult true → specific response + mock.on( + { userMessage: "combined", turnIndex: 1, hasToolResult: true }, + { content: "combined-specific" }, + ); + // Fallback: just userMessage → matches anything else + mock.on({ userMessage: "combined" }, { content: "combined-fallback" }); + + // Request with turnIndex=1 but NO tool result → AND fails → fallback + const res0 = await chatPost(mock.url, [ + { role: "user", content: "combined" }, + { role: "assistant", content: "something" }, + ]); + expect(res0.status).toBe(200); + const body0 = (await res0.json()) as { choices: { message: { content: string } }[] }; + expect(body0.choices[0].message.content).toBe("combined-fallback"); + + // Request with turnIndex=0 and hasToolResult=true → AND fails (turnIndex wrong) → fallback + const res1 = await chatPost(mock.url, [ + { role: "user", content: "combined" }, + { role: "tool", tool_call_id: "call_1", content: "result" }, + ]); + expect(res1.status).toBe(200); + const body1 = (await res1.json()) as { choices: { message: { content: string } }[] }; + expect(body1.choices[0].message.content).toBe("combined-fallback"); + + // Request with turnIndex=1 AND hasToolResult=true → both pass → specific + const res2 = await chatPost(mock.url, [ + { role: "user", content: "combined" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_c1", + type: "function", + function: { name: "do_thing", arguments: "{}" }, + }, + ], + }, + { role: "tool", tool_call_id: "call_c1", content: "done" }, + ]); + expect(res2.status).toBe(200); + const body2 = (await res2.json()) as { choices: { message: { content: string } }[] }; + expect(body2.choices[0].message.content).toBe("combined-specific"); + }); +}); + +// --------------------------------------------------------------------------- +// 4. JSON fixture loading +// --------------------------------------------------------------------------- + +describe("turnIndex and hasToolResult via JSON fixtures", () => { + let mock: LLMock; + + beforeAll(async () => { + mock = new LLMock(); + await mock.start(); + }); + + afterAll(async () => { + await mock.stop(); + }); + + it("addFixturesFromJSON loads turnIndex and hasToolResult correctly", async () => { + mock.reset(); + mock.addFixturesFromJSON([ + { + match: { userMessage: "json-turn", turnIndex: 0, hasToolResult: false }, + response: { + toolCalls: [{ name: "json_tool", arguments: '{"key":"val"}', id: "call_jt1" }], + }, + }, + { + match: { userMessage: "json-turn", turnIndex: 1, hasToolResult: true }, + response: { content: "json-turn-1-with-tool" }, + }, + ]); + + // Turn 0, no tool result → matches first fixture + const res0 = await chatPost(mock.url, [{ role: "user", content: "json-turn" }]); + expect(res0.status).toBe(200); + const body0 = (await res0.json()) as { + choices: { message: { tool_calls: { function: { name: string } }[] } }[]; + }; + expect(body0.choices[0].message.tool_calls[0].function.name).toBe("json_tool"); + + // Turn 1, with tool result → matches second fixture + const res1 = await chatPost(mock.url, [ + { role: "user", content: "json-turn" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_jt1", + type: "function", + function: { name: "json_tool", arguments: '{"key":"val"}' }, + }, + ], + }, + { role: "tool", tool_call_id: "call_jt1", content: '{"result":"ok"}' }, + ]); + expect(res1.status).toBe(200); + const body1 = (await res1.json()) as { choices: { message: { content: string } }[] }; + expect(body1.choices[0].message.content).toBe("json-turn-1-with-tool"); + }); +}); + +// --------------------------------------------------------------------------- +// 5. turnIndex with onTurn convenience method +// --------------------------------------------------------------------------- + +describe("onTurn convenience method", () => { + let mock: LLMock; + + beforeAll(async () => { + mock = new LLMock(); + await mock.start(); + }); + + afterAll(async () => { + await mock.stop(); + }); + + it("onTurn(n, pattern, response) sets turnIndex correctly", async () => { + mock.reset(); + mock.onTurn(0, "convenient", { content: "on-turn-0" }); + mock.onTurn(1, "convenient", { content: "on-turn-1" }); + + // Turn 0 + const res0 = await chatPost(mock.url, [{ role: "user", content: "convenient" }]); + expect(res0.status).toBe(200); + const body0 = (await res0.json()) as { choices: { message: { content: string } }[] }; + expect(body0.choices[0].message.content).toBe("on-turn-0"); + + // Turn 1 + const res1 = await chatPost(mock.url, [ + { role: "user", content: "convenient" }, + { role: "assistant", content: "on-turn-0" }, + ]); + expect(res1.status).toBe(200); + const body1 = (await res1.json()) as { choices: { message: { content: string } }[] }; + expect(body1.choices[0].message.content).toBe("on-turn-1"); + }); +}); + +// --------------------------------------------------------------------------- +// 6. turnIndex does NOT interfere with sequenceIndex +// --------------------------------------------------------------------------- + +describe("turnIndex independence from sequenceIndex", () => { + let mock: LLMock; + + beforeAll(async () => { + mock = new LLMock(); + await mock.start(); + }); + + afterAll(async () => { + await mock.stop(); + }); + + it("turnIndex is stateless; same request always matches the same turnIndex fixture", async () => { + mock.reset(); + mock.on({ userMessage: "no-drift", turnIndex: 0 }, { content: "always-zero" }); + + // Send the same request 3 times — turnIndex is determined by message + // content (assistant count = 0), so it always matches turnIndex 0 + for (let i = 0; i < 3; i++) { + const res = await chatPost(mock.url, [{ role: "user", content: "no-drift" }]); + expect(res.status).toBe(200); + const body = (await res.json()) as { choices: { message: { content: string } }[] }; + expect(body.choices[0].message.content).toBe("always-zero"); + } + }); +}); From 96cbc8ce23dd56d85c993710e0675ea80dd19660 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Tue, 28 Apr 2026 18:02:12 -0700 Subject: [PATCH 3/4] docs: document turnIndex and hasToolResult match criteria --- README.md | 2 +- docs/index.html | 95 +++++++++++++++++++++++++ skills/write-fixtures/SKILL.md | 124 ++++++++++++++++++++------------- 3 files changed, 171 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index f85b19d1..561d185f 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Run them all on one port with `npx @copilotkit/aimock --config aimock.json`, or ## Features - **[Record & Replay](https://aimock.copilotkit.dev/record-replay)** — Proxy real APIs, save as fixtures, replay deterministically forever -- **[Multi-turn Conversations](https://aimock.copilotkit.dev/multi-turn)** — Record and replay multi-turn traces with tool rounds; match distinct turns via `toolCallId`, `sequenceIndex`, or custom predicates +- **[Multi-turn Conversations](https://aimock.copilotkit.dev/multi-turn)** — Record and replay multi-turn traces with tool rounds; match distinct turns via `turnIndex`, `hasToolResult`, `toolCallId`, `sequenceIndex`, or custom predicates - **[11 LLM Providers](https://aimock.copilotkit.dev/docs)** — OpenAI Chat, OpenAI Responses, OpenAI Realtime, Claude, Gemini, Gemini Live, Azure, Bedrock, Vertex AI, Ollama, Cohere — full streaming support - **Multimedia APIs** — [image generation](https://aimock.copilotkit.dev/images) (DALL-E, Imagen), [text-to-speech](https://aimock.copilotkit.dev/speech), [audio transcription](https://aimock.copilotkit.dev/transcription), [video generation](https://aimock.copilotkit.dev/video) - **[MCP](https://aimock.copilotkit.dev/mcp-mock) / [A2A](https://aimock.copilotkit.dev/a2a-mock) / [AG-UI](https://aimock.copilotkit.dev/agui-mock) / [Vector](https://aimock.copilotkit.dev/vector-mock)** — Mock every protocol your AI agents use diff --git a/docs/index.html b/docs/index.html index 2e32da81..3763d7a9 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1069,6 +1069,66 @@ text-decoration: none; } + /* --- Cross-Nav Footer -------------------------------------------- */ + .cross-nav-footer { + padding: 2rem 0; + border-top: 1px solid var(--border); + } + .cross-nav-toolbar { + max-width: 1120px; + margin: 0 auto; + padding: 0 2rem; + display: flex; + align-items: center; + justify-content: center; + gap: 0.5rem; + flex-wrap: wrap; + } + .cn-label { + font-family: "JetBrains Mono", "SF Mono", "Fira Code", monospace; + font-size: 0.8rem; + color: var(--text-dim); + margin-right: 0.25rem; + white-space: nowrap; + } + .cn-pill { + display: inline-flex; + align-items: center; + gap: 0.4rem; + padding: 0.35rem 0.85rem; + border: 1px solid var(--border); + border-radius: 100px; + font-family: "JetBrains Mono", "SF Mono", "Fira Code", monospace; + font-size: 0.75rem; + font-weight: 500; + color: var(--text-secondary); + text-decoration: none; + transition: + border-color 0.2s, + background 0.2s, + color 0.2s; + white-space: nowrap; + } + .cn-pill:hover { + border-color: #3a3a50; + color: var(--text-primary); + text-decoration: none; + background: rgba(255, 255, 255, 0.03); + } + .cn-pill.active { + border-color: var(--accent); + background: var(--accent-glow); + color: var(--text-primary); + cursor: default; + pointer-events: none; + } + .cn-dot { + width: 6px; + height: 6px; + border-radius: 50%; + flex-shrink: 0; + } + /* ─── Mobile ──────────────────────────────────────────────────── */ @media (max-width: 768px) { .nav-links { @@ -1111,6 +1171,16 @@ } } + @media (max-width: 640px) { + .cross-nav-toolbar { + gap: 0.4rem; + } + .cn-pill { + font-size: 0.7rem; + padding: 0.3rem 0.7rem; + } + } + @media (max-width: 480px) { .hero { padding: 8rem 0 4rem; @@ -1121,6 +1191,9 @@ .nav-brand .powered-by { display: none; } + .cn-label { + display: none; + } } @@ -1922,6 +1995,28 @@

Built for production

+ +
+
+ copilotkit.dev / + + vscode + + + aimock + + + oversight + + + pathfinder + + + outpost + +
+
+