From b5a499ce1b2669be1b215035f698cbd33ea003f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 08:41:31 +0000 Subject: [PATCH 1/2] test: property-check model-map resolution invariants Six fast-check properties over the model resolution pipeline (resolveNormalizedModel / getModelProfile / getNormalizedModel), generating known aliases, synthesized GPT-5 spellings (minors 0-9, mini/nano/pro/codex variants, -/./space separators, suffixes), provider prefixes, random casing, and raw garbage: - closed world: every resolution lands on a MODEL_PROFILES key, so getModelProfile's DEFAULT_MODEL fallback is pure defence - idempotence: normalized outputs are fixpoints - provider prefixes and casing never change the resolution - codex dominance: unmapped ids mentioning codex resolve to CURRENT_CODEX_MODEL, never a general model - the inverse: unmapped general GPT-5 spellings stay codex-free and in the gpt-5 family (no silent codex routing for general requests) - every explicit MODEL_MAP alias resolves to its mapped target under any prefix/casing spelling Companion to the property suites in #574/#575/#592-#596. https://claude.ai/code/session_01XNtnkLbBiXZxfQQYLMpucB --- test/property/model-map.property.test.ts | 133 +++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 test/property/model-map.property.test.ts diff --git a/test/property/model-map.property.test.ts b/test/property/model-map.property.test.ts new file mode 100644 index 00000000..e8e89d11 --- /dev/null +++ b/test/property/model-map.property.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from "vitest"; +import * as fc from "fast-check"; +import { + CURRENT_CODEX_MODEL, + DEFAULT_MODEL, + getModelProfile, + getNormalizedModel, + MODEL_MAP, + MODEL_PROFILES, + resolveNormalizedModel, +} from "../../lib/request/helpers/model-map.js"; + +const PROFILE_KEYS = new Set(Object.keys(MODEL_PROFILES)); +const MAP_KEYS = Object.keys(MODEL_MAP); + +// Plausible model spellings: known aliases, synthesized GPT-5 family ids with +// varied separators/variants, codex spellings, and outright garbage. +const arbSynthesizedGpt5 = fc + .record({ + minor: fc.option(fc.integer({ min: 0, max: 9 }), { nil: undefined }), + variant: fc.constantFrom("", "mini", "nano", "pro", "codex"), + separator: fc.constantFrom("-", ".", " "), + suffix: fc.constantFrom("", "-latest", "-2026"), + }) + .map(({ minor, variant, separator, suffix }) => { + const base = minor === undefined ? "gpt-5" : `gpt-5.${minor}`; + const withVariant = variant ? `${base}${separator}${variant}` : base; + return `${withVariant}${suffix}`; + }); + +const arbModelId = fc.oneof( + fc.constantFrom(...MAP_KEYS), + arbSynthesizedGpt5, + fc.string({ maxLength: 24 }), +); + +const arbPrefix = fc.constantFrom("", "openai/", "models/", "providers/openai/"); + +function randomizeCase(value: string, flips: boolean[]): string { + return [...value] + .map((char, index) => + flips[index % Math.max(1, flips.length)] + ? char.toUpperCase() + : char.toLowerCase(), + ) + .join(""); +} + +describe("model-map resolution property invariants", () => { + it("resolveNormalizedModel always lands on a model with a profile", () => { + fc.assert( + fc.property(arbPrefix, arbModelId, (prefix, modelId) => { + const resolved = resolveNormalizedModel(`${prefix}${modelId}`); + // Closed world: whatever the input, the effective model must have a + // profile entry — getModelProfile's DEFAULT_MODEL fallback exists + // for defence, but no reachable resolution should need it. + expect(PROFILE_KEYS.has(resolved)).toBe(true); + expect(getModelProfile(`${prefix}${modelId}`)).toBe( + MODEL_PROFILES[resolved], + ); + }), + ); + }); + + it("resolution is idempotent: normalized outputs are fixpoints", () => { + fc.assert( + fc.property(arbPrefix, arbModelId, (prefix, modelId) => { + const once = resolveNormalizedModel(`${prefix}${modelId}`); + expect(resolveNormalizedModel(once)).toBe(once); + }), + ); + }); + + it("provider prefixes and casing never change the resolution", () => { + fc.assert( + fc.property( + arbModelId, + arbPrefix, + fc.array(fc.boolean(), { minLength: 1, maxLength: 8 }), + (modelId, prefix, flips) => { + const plain = resolveNormalizedModel(modelId); + expect(resolveNormalizedModel(`${prefix}${modelId}`)).toBe(plain); + expect(resolveNormalizedModel(randomizeCase(modelId, flips))).toBe( + plain, + ); + }, + ), + ); + }); + + it("unmapped ids mentioning codex resolve to the current codex model, never a general one", () => { + fc.assert( + fc.property(arbSynthesizedGpt5, fc.constantFrom("-", " "), (modelId, sep) => { + const codexId = modelId.includes("codex") + ? modelId + : `${modelId}${sep}codex`; + fc.pre(getNormalizedModel(codexId) === undefined); + expect(resolveNormalizedModel(codexId)).toBe(CURRENT_CODEX_MODEL); + }), + ); + }); + + it("unmapped general GPT-5 spellings stay in the general family, codex-free", () => { + fc.assert( + fc.property(arbSynthesizedGpt5, (modelId) => { + fc.pre(!modelId.includes("codex")); + fc.pre(getNormalizedModel(modelId) === undefined); + const resolved = resolveNormalizedModel(modelId); + // A general-purpose GPT-5 request must never silently route to a + // codex-tuned model (the inverse of the codex-dominance rule). + expect(resolved.includes("codex")).toBe(false); + expect(resolved.startsWith("gpt-5")).toBe(true); + }), + ); + }); + + it("every explicit alias resolves to its mapped target under any spelling", () => { + fc.assert( + fc.property( + fc.constantFrom(...MAP_KEYS), + arbPrefix, + fc.array(fc.boolean(), { minLength: 1, maxLength: 8 }), + (alias, prefix, flips) => { + const target = MODEL_MAP[alias]; + expect(resolveNormalizedModel(`${prefix}${alias}`)).toBe(target); + expect( + resolveNormalizedModel(randomizeCase(`${prefix}${alias}`, flips)), + ).toBe(target); + }, + ), + ); + }); +}); From 15447c27be7d37260183a720d28ed607f4f173af Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 11 Jun 2026 08:49:56 +0000 Subject: [PATCH 2/2] test: combine prefix and casing mutation in the invariance property https://claude.ai/code/session_01XNtnkLbBiXZxfQQYLMpucB --- test/property/model-map.property.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/property/model-map.property.test.ts b/test/property/model-map.property.test.ts index e8e89d11..8f1068d4 100644 --- a/test/property/model-map.property.test.ts +++ b/test/property/model-map.property.test.ts @@ -83,6 +83,11 @@ describe("model-map resolution property invariants", () => { expect(resolveNormalizedModel(randomizeCase(modelId, flips))).toBe( plain, ); + // Combined pressure: prefix AND casing mutated together, so the + // strip-then-fold pipeline is exercised as one path. + expect( + resolveNormalizedModel(randomizeCase(`${prefix}${modelId}`, flips)), + ).toBe(plain); }, ), );