Skip to content

Commit d362f63

Browse files
jpr5jpr5
committed
feat: add turnIndex and hasToolResult stateless match criteria
turnIndex counts assistant messages in the request conversation history. hasToolResult checks for the presence of tool-role messages. Both are stateless and safe for shared aimock instances with concurrent clients. Includes onTurn() convenience method and unit tests.
1 parent a24e2db commit d362f63

6 files changed

Lines changed: 229 additions & 5 deletions

File tree

‎src/__tests__/router.test.ts‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -582,6 +582,172 @@ describe("matchFixture — sequenceIndex", () => {
582582
});
583583
});
584584

585+
// ---------------------------------------------------------------------------
586+
// matchFixture — turnIndex
587+
// ---------------------------------------------------------------------------
588+
589+
describe("matchFixture — turnIndex", () => {
590+
it("matches when assistant message count equals turnIndex", () => {
591+
const fixture = makeFixture({ userMessage: "hello", turnIndex: 1 });
592+
const req = makeReq({
593+
messages: [
594+
{ role: "system", content: "you are helpful" },
595+
{ role: "user", content: "hello" },
596+
{ role: "assistant", content: "hi there" },
597+
{ role: "user", content: "hello" },
598+
],
599+
});
600+
expect(matchFixture([fixture], req)).toBe(fixture);
601+
});
602+
603+
it("skips when assistant message count does not equal turnIndex", () => {
604+
const fixture = makeFixture({ userMessage: "hello", turnIndex: 2 });
605+
const req = makeReq({
606+
messages: [
607+
{ role: "user", content: "hello" },
608+
{ role: "assistant", content: "hi there" },
609+
{ role: "user", content: "hello" },
610+
],
611+
});
612+
expect(matchFixture([fixture], req)).toBeNull();
613+
});
614+
615+
it("turnIndex 0 matches when no assistant messages present", () => {
616+
const fixture = makeFixture({ userMessage: "hello", turnIndex: 0 });
617+
const req = makeReq({
618+
messages: [
619+
{ role: "system", content: "you are helpful" },
620+
{ role: "user", content: "hello" },
621+
],
622+
});
623+
expect(matchFixture([fixture], req)).toBe(fixture);
624+
});
625+
626+
it("selects correct fixture from turnIndex sequence", () => {
627+
const turn0 = makeFixture({ userMessage: "hello", turnIndex: 0 }, { content: "turn-0" });
628+
const turn1 = makeFixture({ userMessage: "hello", turnIndex: 1 }, { content: "turn-1" });
629+
const turn2 = makeFixture({ userMessage: "hello", turnIndex: 2 }, { content: "turn-2" });
630+
631+
const req0 = makeReq({
632+
messages: [{ role: "user", content: "hello" }],
633+
});
634+
expect(matchFixture([turn0, turn1, turn2], req0)).toBe(turn0);
635+
636+
const req1 = makeReq({
637+
messages: [
638+
{ role: "user", content: "hello" },
639+
{ role: "assistant", content: "reply" },
640+
{ role: "user", content: "hello" },
641+
],
642+
});
643+
expect(matchFixture([turn0, turn1, turn2], req1)).toBe(turn1);
644+
645+
const req2 = makeReq({
646+
messages: [
647+
{ role: "user", content: "hello" },
648+
{ role: "assistant", content: "reply1" },
649+
{ role: "user", content: "hello" },
650+
{ role: "assistant", content: "reply2" },
651+
{ role: "user", content: "hello" },
652+
],
653+
});
654+
expect(matchFixture([turn0, turn1, turn2], req2)).toBe(turn2);
655+
});
656+
657+
it("falls through to non-turnIndex fixture when no turnIndex matches", () => {
658+
const turnOnly = makeFixture({ userMessage: "hello", turnIndex: 0 }, { content: "turn-0" });
659+
const fallback = makeFixture({ userMessage: "hello" }, { content: "fallback" });
660+
const req = makeReq({
661+
messages: [
662+
{ role: "user", content: "hello" },
663+
{ role: "assistant", content: "reply1" },
664+
{ role: "user", content: "hello" },
665+
{ role: "assistant", content: "reply2" },
666+
{ role: "user", content: "hello" },
667+
],
668+
});
669+
expect(matchFixture([turnOnly, fallback], req)).toBe(fallback);
670+
});
671+
});
672+
673+
// ---------------------------------------------------------------------------
674+
// matchFixture — hasToolResult
675+
// ---------------------------------------------------------------------------
676+
677+
describe("matchFixture — hasToolResult", () => {
678+
it("matches hasToolResult: true when tool messages present", () => {
679+
const fixture = makeFixture({ userMessage: "hello", hasToolResult: true });
680+
const req = makeReq({
681+
messages: [
682+
{ role: "user", content: "hello" },
683+
{ role: "assistant", content: "calling tool" },
684+
{ role: "tool", content: "tool output" },
685+
{ role: "user", content: "hello" },
686+
],
687+
});
688+
expect(matchFixture([fixture], req)).toBe(fixture);
689+
});
690+
691+
it("skips hasToolResult: true when no tool messages present", () => {
692+
const fixture = makeFixture({ userMessage: "hello", hasToolResult: true });
693+
const req = makeReq({
694+
messages: [
695+
{ role: "user", content: "hello" },
696+
{ role: "assistant", content: "reply" },
697+
{ role: "user", content: "hello" },
698+
],
699+
});
700+
expect(matchFixture([fixture], req)).toBeNull();
701+
});
702+
703+
it("matches hasToolResult: false when no tool messages present", () => {
704+
const fixture = makeFixture({ userMessage: "hello", hasToolResult: false });
705+
const req = makeReq({
706+
messages: [{ role: "user", content: "hello" }],
707+
});
708+
expect(matchFixture([fixture], req)).toBe(fixture);
709+
});
710+
711+
it("skips hasToolResult: false when tool messages present", () => {
712+
const fixture = makeFixture({ userMessage: "hello", hasToolResult: false });
713+
const req = makeReq({
714+
messages: [
715+
{ role: "user", content: "hello" },
716+
{ role: "assistant", content: "calling tool" },
717+
{ role: "tool", content: "tool output" },
718+
{ role: "user", content: "hello" },
719+
],
720+
});
721+
expect(matchFixture([fixture], req)).toBeNull();
722+
});
723+
724+
it("discriminates 2-step HITL flow with hasToolResult", () => {
725+
const beforeTool = makeFixture(
726+
{ userMessage: "hello", hasToolResult: false },
727+
{ content: "before-tool" },
728+
);
729+
const afterTool = makeFixture(
730+
{ userMessage: "hello", hasToolResult: true },
731+
{ content: "after-tool" },
732+
);
733+
734+
const reqBefore = makeReq({
735+
messages: [{ role: "user", content: "hello" }],
736+
});
737+
expect(matchFixture([beforeTool, afterTool], reqBefore)).toBe(beforeTool);
738+
739+
const reqAfter = makeReq({
740+
messages: [
741+
{ role: "user", content: "hello" },
742+
{ role: "assistant", content: "calling tool" },
743+
{ role: "tool", content: "result" },
744+
{ role: "user", content: "hello" },
745+
],
746+
});
747+
expect(matchFixture([beforeTool, afterTool], reqAfter)).toBe(afterTool);
748+
});
749+
});
750+
585751
// ---------------------------------------------------------------------------
586752
// matchFixture — first-match-wins
587753
// ---------------------------------------------------------------------------

‎src/fixture-loader.ts‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,12 @@ export function entryToFixture(entry: FixtureFileEntry): Fixture {
5959
responseFormat: entry.match.responseFormat,
6060
endpoint: entry.match.endpoint,
6161
...(entry.match.sequenceIndex !== undefined && { sequenceIndex: entry.match.sequenceIndex }),
62+
...(entry.match.turnIndex !== undefined && {
63+
turnIndex: entry.match.turnIndex,
64+
}),
65+
...(entry.match.hasToolResult !== undefined && {
66+
hasToolResult: entry.match.hasToolResult,
67+
}),
6268
},
6369
response: normalizeResponse(entry.response),
6470
...(entry.latency !== undefined && { latency: entry.latency }),
@@ -525,20 +531,45 @@ export function validateFixtures(fixtures: Fixture[]): ValidationResult[] {
525531
}
526532
}
527533

534+
// Match field type checks
535+
if (f.match.turnIndex !== undefined) {
536+
if (
537+
typeof f.match.turnIndex !== "number" ||
538+
f.match.turnIndex < 0 ||
539+
!Number.isInteger(f.match.turnIndex)
540+
) {
541+
results.push({
542+
severity: "error",
543+
fixtureIndex: i,
544+
message: "match.turnIndex must be a non-negative integer",
545+
});
546+
}
547+
}
548+
if (f.match.hasToolResult !== undefined && typeof f.match.hasToolResult !== "boolean") {
549+
results.push({
550+
severity: "error",
551+
fixtureIndex: i,
552+
message: `match.hasToolResult must be a boolean, got ${typeof f.match.hasToolResult}`,
553+
});
554+
}
555+
528556
// --- Warning checks ---
529557

530-
// Duplicate userMessage shadowing
558+
// Duplicate userMessage shadowing — include turnIndex, hasToolResult, and
559+
// sequenceIndex in the dedup key so that fixtures which share a userMessage
560+
// but differ on those fields are NOT considered duplicates.
531561
const um = f.match.userMessage;
532562
if (typeof um === "string" && um) {
533-
const prev = seenUserMessages.get(um);
563+
const dedupKey = `${um}|${f.match.turnIndex}|${f.match.hasToolResult}|${f.match.sequenceIndex}`;
564+
const prev = seenUserMessages.get(dedupKey);
534565
if (prev !== undefined) {
535566
results.push({
536567
severity: "warning",
537568
fixtureIndex: i,
538569
message: `duplicate userMessage '${um}' — shadows fixture ${prev}`,
539570
});
540571
} else {
541-
seenUserMessages.set(um, i);
572+
seenUserMessages.set(dedupKey, i);
542573
}
543574
}
544575

@@ -552,7 +583,9 @@ export function validateFixtures(fixtures: Fixture[]): ValidationResult[] {
552583
match.toolCallId !== undefined ||
553584
match.toolName !== undefined ||
554585
match.model !== undefined ||
555-
match.predicate !== undefined;
586+
match.predicate !== undefined ||
587+
match.turnIndex !== undefined ||
588+
match.hasToolResult !== undefined;
556589

557590
if (!hasDiscriminator && i < fixtures.length - 1) {
558591
results.push({

‎src/journal.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@ function matchCriteriaEqual(a: FixtureMatch, b: FixtureMatch): boolean {
2626
fieldEqual(a.model, b.model) &&
2727
fieldEqual(a.responseFormat, b.responseFormat) &&
2828
fieldEqual(a.predicate, b.predicate) &&
29-
fieldEqual(a.endpoint, b.endpoint)
29+
fieldEqual(a.endpoint, b.endpoint) &&
30+
fieldEqual(a.turnIndex, b.turnIndex) &&
31+
fieldEqual(a.hasToolResult, b.hasToolResult)
3032
);
3133
}
3234

‎src/llmock.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,15 @@ export class LLMock {
130130
return this.on({ toolCallId: id }, response, opts);
131131
}
132132

133+
onTurn(
134+
turn: number,
135+
pattern: string | RegExp,
136+
response: FixtureFileResponse,
137+
opts?: FixtureOpts,
138+
): this {
139+
return this.on({ userMessage: pattern, turnIndex: turn }, response, opts);
140+
}
141+
133142
onImage(prompt: string | RegExp, response: ImageResponse): this {
134143
return this.addFixture({
135144
match: { userMessage: prompt, endpoint: "image" },

‎src/router.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,16 @@ export function matchFixture(
140140
if (count !== match.sequenceIndex) continue;
141141
}
142142

143+
if (match.turnIndex !== undefined) {
144+
const assistantCount = effective.messages.filter((m) => m.role === "assistant").length;
145+
if (assistantCount !== match.turnIndex) continue;
146+
}
147+
148+
if (match.hasToolResult !== undefined) {
149+
const hasTool = effective.messages.some((m) => m.role === "tool");
150+
if (hasTool !== match.hasToolResult) continue;
151+
}
152+
143153
return fixture;
144154
}
145155

‎src/types.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ export interface FixtureMatch {
7272
predicate?: (req: ChatCompletionRequest) => boolean;
7373
/** Which occurrence of this match to respond to (0-indexed). Undefined means match any. */
7474
sequenceIndex?: number;
75+
turnIndex?: number;
76+
hasToolResult?: boolean;
7577
endpoint?: "chat" | "image" | "speech" | "transcription" | "video" | "embedding";
7678
}
7779

@@ -277,6 +279,8 @@ export interface FixtureFileEntry {
277279
model?: string;
278280
responseFormat?: string;
279281
sequenceIndex?: number;
282+
turnIndex?: number;
283+
hasToolResult?: boolean;
280284
endpoint?: "chat" | "image" | "speech" | "transcription" | "video" | "embedding";
281285
// predicate not supported in JSON files
282286
};

0 commit comments

Comments
 (0)