|
| 1 | +import { describe, it, test, expect, afterEach } from "vitest"; |
| 2 | +import * as http from "node:http"; |
| 3 | +import * as fs from "node:fs"; |
| 4 | +import * as os from "node:os"; |
| 5 | +import * as path from "node:path"; |
| 6 | +import { matchFixture, matchFixtureDiagnostic, getTextContent, getSystemText } from "../router.js"; |
| 7 | +import { LLMock } from "../llmock.js"; |
| 8 | +import type { ChatCompletionRequest, Fixture } from "../types.js"; |
| 9 | + |
| 10 | +// =========================================================================== |
| 11 | +// CR fixes for the content-anchored fixture-matching change. One concern: |
| 12 | +// the content-anchored selection logic + record-path wiring must be correct. |
| 13 | +// =========================================================================== |
| 14 | + |
| 15 | +function makeReq(overrides: Partial<ChatCompletionRequest> = {}): ChatCompletionRequest { |
| 16 | + return { |
| 17 | + model: "gpt-4o", |
| 18 | + messages: [{ role: "user", content: "hello" }], |
| 19 | + ...overrides, |
| 20 | + }; |
| 21 | +} |
| 22 | + |
| 23 | +function makeFixture( |
| 24 | + match: Fixture["match"], |
| 25 | + response: Fixture["response"] = { content: "ok" }, |
| 26 | +): Fixture { |
| 27 | + return { match, response }; |
| 28 | +} |
| 29 | + |
| 30 | +// --------------------------------------------------------------------------- |
| 31 | +// F2 — selectByTurnIndex asymmetry + registration-order break |
| 32 | +// --------------------------------------------------------------------------- |
| 33 | + |
| 34 | +describe("F2: future-turn guard applied uniformly (single + multi candidate)", () => { |
| 35 | + it("single content-matching fixture whose turnIndex is AHEAD of the conversation does not answer an at-turn-0 request via the relaxed path", () => { |
| 36 | + // A lone candidate at turnIndex 3 must NOT answer an at-turn-0 request when |
| 37 | + // there is no other eligible candidate — same future-turn guard the |
| 38 | + // multi-candidate path enforces. (Replay: such a fixture is the only |
| 39 | + // content match, so the contract DOES serve it; this asserts that the |
| 40 | + // single-candidate path and the multi-candidate path agree on a request |
| 41 | + // that has a non-turn fallback alternative.) |
| 42 | + const fixtures = [ |
| 43 | + makeFixture({ userMessage: "step", turnIndex: 3 }, { content: "future" }), |
| 44 | + makeFixture({ userMessage: "step" }, { content: "fallback" }), |
| 45 | + ]; |
| 46 | + // assistantCount 0; turnIndex 3 is ahead → the non-turn fallback must win, |
| 47 | + // not the future-turn fixture. |
| 48 | + const got = matchFixture(fixtures, makeReq({ messages: [{ role: "user", content: "step" }] })); |
| 49 | + expect(got?.response).toEqual({ content: "fallback" }); |
| 50 | + }); |
| 51 | +}); |
| 52 | + |
| 53 | +describe("F2: registration order preserved among equally-eligible candidates", () => { |
| 54 | + it("a later-registered turnIndex'd fixture does NOT override an earlier-registered non-turnIndex'd one when both are eligible", () => { |
| 55 | + // Both are content matches and both are eligible at assistantCount 0 |
| 56 | + // (turnIndex 0 <= 0). The first-registered fixture must win (registration |
| 57 | + // order tie-break), regardless of which one carries a turnIndex. |
| 58 | + const fixtures = [ |
| 59 | + makeFixture({ userMessage: "tie" }, { content: "first-registered" }), |
| 60 | + makeFixture({ userMessage: "tie", turnIndex: 0 }, { content: "second-registered" }), |
| 61 | + ]; |
| 62 | + const got = matchFixture(fixtures, makeReq({ messages: [{ role: "user", content: "tie" }] })); |
| 63 | + expect(got?.response).toEqual({ content: "first-registered" }); |
| 64 | + }); |
| 65 | +}); |
| 66 | + |
| 67 | +// --------------------------------------------------------------------------- |
| 68 | +// F3 — fallback must not serve a future-turn fixture |
| 69 | +// --------------------------------------------------------------------------- |
| 70 | + |
| 71 | +describe("F3: fallback does not serve a future-turn fixture to an at-turn-0 request", () => { |
| 72 | + it("a turn-3 fixture must not answer an at-turn-0 request when a fallback alternative exists", () => { |
| 73 | + const fixtures = [ |
| 74 | + makeFixture({ userMessage: "go", turnIndex: 3 }, { content: "turn-3" }), |
| 75 | + makeFixture({ userMessage: "go", turnIndex: 5 }, { content: "turn-5" }), |
| 76 | + makeFixture({ userMessage: "go" }, { content: "plain-fallback" }), |
| 77 | + ]; |
| 78 | + // assistantCount 0; every turnIndexed candidate (3, 5) is ahead → the plain |
| 79 | + // fallback answers, NOT the lowest future turn. |
| 80 | + const got = matchFixture(fixtures, makeReq({ messages: [{ role: "user", content: "go" }] })); |
| 81 | + expect(got?.response).toEqual({ content: "plain-fallback" }); |
| 82 | + }); |
| 83 | +}); |
| 84 | + |
| 85 | +// --------------------------------------------------------------------------- |
| 86 | +// F4 — text-join + empty-handling consistency |
| 87 | +// --------------------------------------------------------------------------- |
| 88 | + |
| 89 | +describe("F4: getTextContent / getSystemText consistent multi-part + empty handling", () => { |
| 90 | + it("empty-string string content and empty-text array content are treated the same (both null/empty)", () => { |
| 91 | + // String "" historically returns "" (skipped via !text); array of only |
| 92 | + // empty text returns null. After the fix both collapse to the same empty |
| 93 | + // semantic so content matching is symmetric. |
| 94 | + const fromString = getTextContent(""); |
| 95 | + const fromArray = getTextContent([{ type: "text", text: "" }]); |
| 96 | + expect(Boolean(fromString)).toBe(false); |
| 97 | + expect(Boolean(fromArray)).toBe(false); |
| 98 | + }); |
| 99 | + |
| 100 | + it("getSystemText joins multi-part text within a single system message the same way getTextContent does", () => { |
| 101 | + const joined = getTextContent([ |
| 102 | + { type: "text", text: "alpha" }, |
| 103 | + { type: "text", text: "beta" }, |
| 104 | + ]); |
| 105 | + const sys = getSystemText([ |
| 106 | + { |
| 107 | + role: "system", |
| 108 | + content: [ |
| 109 | + { type: "text", text: "alpha" }, |
| 110 | + { type: "text", text: "beta" }, |
| 111 | + ], |
| 112 | + }, |
| 113 | + ]); |
| 114 | + // A single system message's parts must read identically through both paths. |
| 115 | + expect(sys).toBe(joined); |
| 116 | + }); |
| 117 | + |
| 118 | + it("systemMessage:[] matches unconditionally even when the request has no system text (F5 fold)", () => { |
| 119 | + const fixtures = [makeFixture({ systemMessage: [] }, { content: "unconditional" })]; |
| 120 | + // No system message at all — the empty-array contract is "no constraint". |
| 121 | + const got = matchFixture(fixtures, makeReq({ messages: [{ role: "user", content: "x" }] })); |
| 122 | + expect(got?.response).toEqual({ content: "unconditional" }); |
| 123 | + }); |
| 124 | +}); |
| 125 | + |
| 126 | +// --------------------------------------------------------------------------- |
| 127 | +// F1 — sequenceIndex consumed by a declined fixture |
| 128 | +// --------------------------------------------------------------------------- |
| 129 | + |
| 130 | +describe("F1: sequence match-count bumps only for the SELECTED fixture", () => { |
| 131 | + it("a sequenced fixture that passes its gate but is NOT served by selectByTurnIndex is not consumed", async () => { |
| 132 | + const mock = new LLMock(); |
| 133 | + await mock.start(); |
| 134 | + try { |
| 135 | + mock.reset(); |
| 136 | + // A turnIndex'd fixture B (registered FIRST so it wins the position tie) |
| 137 | + // AND a sequenced fixture A at sequenceIndex 0 that also content-matches. |
| 138 | + // At assistantCount 1, selectByTurnIndex serves B (turnIndex 1 == count, |
| 139 | + // registered first). A passed its sequence gate (count 0 == index 0) but |
| 140 | + // must NOT have its count consumed, because B — not A — was served. |
| 141 | + mock.on({ userMessage: "seq", turnIndex: 1 }, { content: "B-turn-1" }); |
| 142 | + mock.on({ userMessage: "seq", sequenceIndex: 0 }, { content: "A-seq-0" }); |
| 143 | + mock.on({ userMessage: "seq", sequenceIndex: 1 }, { content: "A-seq-1" }); |
| 144 | + |
| 145 | + // assistantCount 1 → B (turnIndex 1) is the closest scripted turn → served. |
| 146 | + const res1 = await fetch(`${mock.url}/v1/chat/completions`, { |
| 147 | + method: "POST", |
| 148 | + headers: { "Content-Type": "application/json" }, |
| 149 | + body: JSON.stringify({ |
| 150 | + model: "gpt-4", |
| 151 | + stream: false, |
| 152 | + messages: [ |
| 153 | + { role: "user", content: "seq" }, |
| 154 | + { role: "assistant", content: "prior" }, |
| 155 | + { role: "user", content: "seq" }, |
| 156 | + ], |
| 157 | + }), |
| 158 | + }); |
| 159 | + expect(res1.status).toBe(200); |
| 160 | + const body1 = (await res1.json()) as { choices: { message: { content: string } }[] }; |
| 161 | + expect(body1.choices[0].message.content).toBe("B-turn-1"); |
| 162 | + |
| 163 | + // Now an at-turn-0 request: sequence A must STILL be at index 0 (not |
| 164 | + // consumed by the prior request which served B). So we get A-seq-0. |
| 165 | + const res2 = await fetch(`${mock.url}/v1/chat/completions`, { |
| 166 | + method: "POST", |
| 167 | + headers: { "Content-Type": "application/json" }, |
| 168 | + body: JSON.stringify({ |
| 169 | + model: "gpt-4", |
| 170 | + stream: false, |
| 171 | + messages: [{ role: "user", content: "seq" }], |
| 172 | + }), |
| 173 | + }); |
| 174 | + expect(res2.status).toBe(200); |
| 175 | + const body2 = (await res2.json()) as { choices: { message: { content: string } }[] }; |
| 176 | + // If the prior request had wrongly consumed A's index, this would serve |
| 177 | + // A-seq-1 instead. Correct behavior: A is untouched → A-seq-0. |
| 178 | + expect(body2.choices[0].message.content).toBe("A-seq-0"); |
| 179 | + } finally { |
| 180 | + await mock.stop(); |
| 181 | + } |
| 182 | + }); |
| 183 | +}); |
| 184 | + |
| 185 | +// --------------------------------------------------------------------------- |
| 186 | +// F6 — strictTurnIndex wired on ALL record-capable handlers (not just OpenAI) |
| 187 | +// --------------------------------------------------------------------------- |
| 188 | + |
| 189 | +interface FakeUpstream { |
| 190 | + url: string; |
| 191 | + close: () => Promise<void>; |
| 192 | + getHits: () => number; |
| 193 | +} |
| 194 | + |
| 195 | +function startAnthropicUpstream(): Promise<FakeUpstream> { |
| 196 | + let hits = 0; |
| 197 | + return new Promise((resolve) => { |
| 198 | + const server = http.createServer((req, res) => { |
| 199 | + let raw = ""; |
| 200 | + req.on("data", (c) => (raw += c)); |
| 201 | + req.on("end", () => { |
| 202 | + void raw; |
| 203 | + hits++; |
| 204 | + res.writeHead(200, { "Content-Type": "application/json" }); |
| 205 | + res.end( |
| 206 | + JSON.stringify({ |
| 207 | + id: "msg_rec", |
| 208 | + type: "message", |
| 209 | + role: "assistant", |
| 210 | + model: "claude-3-5-sonnet-20241022", |
| 211 | + content: [{ type: "text", text: "recorded-second-turn" }], |
| 212 | + stop_reason: "end_turn", |
| 213 | + usage: { input_tokens: 1, output_tokens: 1 }, |
| 214 | + }), |
| 215 | + ); |
| 216 | + }); |
| 217 | + }); |
| 218 | + server.listen(0, "127.0.0.1", () => { |
| 219 | + const addr = server.address() as { port: number }; |
| 220 | + resolve({ |
| 221 | + url: `http://127.0.0.1:${addr.port}`, |
| 222 | + close: () => |
| 223 | + new Promise<void>((r) => { |
| 224 | + server.close(() => r()); |
| 225 | + }), |
| 226 | + getHits: () => hits, |
| 227 | + }); |
| 228 | + }); |
| 229 | + }); |
| 230 | +} |
| 231 | + |
| 232 | +describe("F6: record mode strictTurnIndex wired on the Anthropic (non-OpenAI) handler", () => { |
| 233 | + let mock: LLMock | undefined; |
| 234 | + let upstream: Awaited<ReturnType<typeof startAnthropicUpstream>> | undefined; |
| 235 | + let tmpDir: string | undefined; |
| 236 | + |
| 237 | + afterEach(async () => { |
| 238 | + await mock?.stop(); |
| 239 | + mock = undefined; |
| 240 | + await upstream?.close(); |
| 241 | + upstream = undefined; |
| 242 | + if (tmpDir) { |
| 243 | + fs.rmSync(tmpDir, { recursive: true, force: true }); |
| 244 | + tmpDir = undefined; |
| 245 | + } |
| 246 | + }); |
| 247 | + |
| 248 | + test("an earlier-turn fixture must NOT shadow a longer record request → the new turn IS proxied/recorded", async () => { |
| 249 | + upstream = await startAnthropicUpstream(); |
| 250 | + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "aimock-anthropic-record-")); |
| 251 | + mock = new LLMock({ |
| 252 | + port: 0, |
| 253 | + record: { providers: { anthropic: upstream!.url }, fixturePath: tmpDir }, |
| 254 | + }); |
| 255 | + await mock.start(); |
| 256 | + |
| 257 | + // A turnIndex-0 fixture that content-matches the user message. A longer |
| 258 | + // (turn-1) request arrives. Under the BUGGY default (strictTurnIndex |
| 259 | + // unset on this handler), the turn-0 fixture content-shadows the longer |
| 260 | + // request → fixture served → recording never fires. With strictTurnIndex |
| 261 | + // wired (record mode), turn-0 != turn-1 → MISS → proxy + record. |
| 262 | + mock.on({ userMessage: "record-me", turnIndex: 0 }, { content: "stale-turn-0" }); |
| 263 | + |
| 264 | + const res = await fetch(`${mock.url}/v1/messages`, { |
| 265 | + method: "POST", |
| 266 | + headers: { |
| 267 | + "Content-Type": "application/json", |
| 268 | + "x-api-key": "test-key", |
| 269 | + "anthropic-version": "2023-06-01", |
| 270 | + }, |
| 271 | + body: JSON.stringify({ |
| 272 | + model: "claude-3-5-sonnet-20241022", |
| 273 | + max_tokens: 1024, |
| 274 | + stream: false, |
| 275 | + messages: [ |
| 276 | + { role: "user", content: "record-me" }, |
| 277 | + { role: "assistant", content: "first turn" }, |
| 278 | + { role: "user", content: "record-me" }, |
| 279 | + ], |
| 280 | + }), |
| 281 | + }); |
| 282 | + |
| 283 | + expect(res.status).toBe(200); |
| 284 | + const body = (await res.json()) as { content: { type: string; text: string }[] }; |
| 285 | + // Must be the freshly recorded upstream turn, NOT the stale turn-0 fixture. |
| 286 | + expect(body.content[0].text).toBe("recorded-second-turn"); |
| 287 | + // Upstream WAS hit (the new turn was proxied + recorded). |
| 288 | + expect(upstream!.getHits()).toBe(1); |
| 289 | + }); |
| 290 | +}); |
| 291 | + |
| 292 | +// --------------------------------------------------------------------------- |
| 293 | +// F6 (unit) — matcher-level proof the shared MatchOptions builder is honored |
| 294 | +// --------------------------------------------------------------------------- |
| 295 | + |
| 296 | +describe("F6 (unit): strictTurnIndex makes an earlier-turn fixture MISS a longer request", () => { |
| 297 | + it("default (false) shadows; strict (true) misses → record branch can fire", () => { |
| 298 | + const fixtures = [makeFixture({ userMessage: "rec", turnIndex: 0 }, { content: "turn-0" })]; |
| 299 | + const longer = makeReq({ |
| 300 | + messages: [ |
| 301 | + { role: "user", content: "rec" }, |
| 302 | + { role: "assistant", content: "a" }, |
| 303 | + { role: "user", content: "rec" }, |
| 304 | + ], |
| 305 | + }); |
| 306 | + // Replay default: the lone content match is served (false-red kill). |
| 307 | + const replayed = matchFixtureDiagnostic(fixtures, longer); |
| 308 | + expect(replayed.fixture).not.toBeNull(); |
| 309 | + // Record (strict): turn-0 != turn-1 → MISS so the handler proxies + records. |
| 310 | + const recorded = matchFixtureDiagnostic(fixtures, longer, undefined, undefined, { |
| 311 | + strictTurnIndex: true, |
| 312 | + }); |
| 313 | + expect(recorded.fixture).toBeNull(); |
| 314 | + }); |
| 315 | +}); |
0 commit comments