Fix model matrix capability handling

Author: ndycode

scripts/test-model-matrix.js

@@ -110,6 +110,7 @@ function printUsage() {
 			"  --plugin=dist|package          Load plugin from local dist URI or package name (default: dist)",
 			"  --max-cases=N                  Hard cap number of cases per scenario",
 			"  --report-json=PATH             Write JSON report to PATH (relative to repo root)",
+			"  --strict-capabilities          Fail unsupported account/model capabilities instead of skipping them",
 			"  --no-restore                   Keep generated local config files after run",
 			"  -h, --help                     Show help",
 		].join("\n"),
@@ -180,18 +181,67 @@ export function resolveMatrixTimeoutMs(smoke = false) {
 	return parsedTimeout;
 }
 
-function hasCompletedSuccessfully(output, token) {
+function parseNdjsonEvents(output) {
+	const events = [];
+	for (const line of output.split(/\r?\n/)) {
+		const trimmed = line.trim();
+		if (!trimmed.startsWith("{")) {
+			continue;
+		}
+		try {
+			events.push(JSON.parse(trimmed));
+		} catch {
+			// Ignore wrapper noise and partial lines.
+		}
+	}
+	return events;
+}
+
+function findLastIndex(items, predicate) {
+	for (let index = items.length - 1; index >= 0; index -= 1) {
+		if (predicate(items[index], index)) {
+			return index;
+		}
+	}
+	return -1;
+}
+
+function hasTerminalFailure(events) {
+	const lastCompletedIndex = findLastIndex(
+		events,
+		(event) =>
+			event?.type === "turn.completed" || event?.type === "response.completed",
+	);
 	return (
-		output.includes(token) ||
-		output.includes('"type":"turn.completed"') ||
-		output.includes('"type":"response.completed"')
+		findLastIndex(
+			events,
+			(event, index) =>
+				index > lastCompletedIndex &&
+				(event?.type === "error" ||
+					event?.type === "turn.failed" ||
+					event?.type === "response.failed" ||
+					event?.type === "response.error" ||
+					event?.type === "response.incomplete"),
+		) >= 0
 	);
 }
 
-function getSmokeSkipReason(exitCode, output) {
-	if (exitCode === 124) {
-		return "timed-out";
+function hasCompletedSuccessfully(output, token) {
+	const events = parseNdjsonEvents(output);
+	if (events.length > 0) {
+		if (hasTerminalFailure(events)) {
+			return false;
+		}
+		return events.some(
+			(event) =>
+				event?.type === "turn.completed" ||
+				event?.type === "response.completed",
+		);
 	}
+	return output.includes(token);
+}
+
+function getCapabilitySkipReason(exitCode, output, smoke) {
 	if (/not supported when using codex with a chatgpt account/i.test(output)) {
 		return "unsupported-model";
 	}
@@ -201,14 +251,26 @@ function getSmokeSkipReason(exitCode, output) {
 	) {
 		return "unsupported-reasoning";
 	}
+	if (smoke && exitCode === 124) {
+		return "timed-out";
+	}
 	return null;
 }
 
-function finalizeModelCaseResult(caseInfo, exitCode, output, token, smoke) {
+function finalizeModelCaseResult(
+	caseInfo,
+	exitCode,
+	output,
+	token,
+	{ smoke, strictCapabilities } = {},
+) {
 	const hasToken = output.includes(token);
 	const completed = hasCompletedSuccessfully(output, token);
 	const ok = exitCode === 0 && completed;
-	const skipReason = !ok && smoke ? getSmokeSkipReason(exitCode, output) : null;
+	const skipReason =
+		!ok && strictCapabilities !== true
+			? getCapabilitySkipReason(exitCode, output, smoke === true)
+			: null;
 
 	return {
 		...caseInfo,
@@ -228,8 +290,12 @@ export function __finalizeModelCaseResultForTests(
 	output,
 	token,
 	smoke = false,
+	strictCapabilities = false,
 ) {
-	return finalizeModelCaseResult(caseInfo, exitCode, output, token, smoke);
+	return finalizeModelCaseResult(caseInfo, exitCode, output, token, {
+		smoke,
+		strictCapabilities,
+	});
 }
 
 function stopCodexServersInternal() {
@@ -361,7 +427,10 @@ function executeModelCase(caseInfo, index) {
 			124,
 			`Timed out after ${timeoutMs}ms`,
 			token,
-			caseInfo.smoke === true,
+			{
+				smoke: caseInfo.smoke === true,
+				strictCapabilities: caseInfo.strictCapabilities === true,
+			},
 		);
 	}
 
@@ -373,7 +442,10 @@ function executeModelCase(caseInfo, index) {
 		exitCode,
 		combinedOutput,
 		token,
-		caseInfo.smoke === true,
+		{
+			smoke: caseInfo.smoke === true,
+			strictCapabilities: caseInfo.strictCapabilities === true,
+		},
 	);
 }
 
@@ -451,6 +523,7 @@ async function runScenario(scenario, options) {
 		(caseInfo) => ({
 			...caseInfo,
 			smoke: options.smoke,
+			strictCapabilities: options.strictCapabilities,
 		}),
 	);
 	console.log(`\n=== ${scenario.toUpperCase()} (${cases.length} cases) ===`);
@@ -492,6 +565,7 @@ async function main() {
 	const scenarioValue =
 		parseArgValue(args, "--scenario") ?? (smoke ? "modern" : "all");
 	const pluginMode = parseArgValue(args, "--plugin") ?? "dist";
+	const strictCapabilities = args.includes("--strict-capabilities");
 	const noRestore = args.includes("--no-restore");
 	const maxCasesRaw = parseArgValue(args, "--max-cases");
 	const maxCases = maxCasesRaw ? Number.parseInt(maxCasesRaw, 10) : 0;
@@ -539,6 +613,7 @@ async function main() {
 				smoke,
 				maxCases,
 				pluginRef,
+				strictCapabilities,
 			});
 			allResults.push(
 				...scenarioResults.map((item) => ({ ...item, scenario })),

test/model-capability-matrix.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, it } from "vitest";
 import { CapabilityPolicyStore } from "../lib/capability-policy.js";
+import { resolveEntitlementAccountKey } from "../lib/entitlement-cache.js";
 import { buildModelCapabilityMatrix } from "../lib/model-capability-matrix.js";
 import type { AccountStorageV3 } from "../lib/storage.js";
 
@@ -21,6 +22,18 @@ function storage(): AccountStorageV3 {
 }
 
 describe("model capability matrix", () => {
+	it("returns default normalized models without entries when storage is missing", () => {
+		const matrix = buildModelCapabilityMatrix({
+			storage: null,
+			models: [],
+			now: 100,
+		});
+
+		expect(matrix.generatedAt).toBe(100);
+		expect(matrix.models.length).toBeGreaterThan(0);
+		expect(matrix.entries).toEqual([]);
+	});
+
 	it("builds model/account availability from existing model profiles", () => {
 		const matrix = buildModelCapabilityMatrix({
 			storage: storage(),
@@ -69,5 +82,46 @@ describe("model capability matrix", () => {
 		);
 		expect(matrix.entries[0]?.reasons).toContain("quota cache is rate-limited");
 	});
+
+	it("marks disabled and entitlement-blocked accounts unavailable", () => {
+		const baseStorage = storage();
+		baseStorage.accounts[0] = {
+			...baseStorage.accounts[0]!,
+			enabled: false,
+		};
+		const entitlementKey = resolveEntitlementAccountKey({
+			accountId: "acct_1",
+			email: "owner@example.com",
+			index: 0,
+		});
+		const matrix = buildModelCapabilityMatrix({
+			storage: baseStorage,
+			models: ["gpt-5.3-codex"],
+			entitlements: {
+				accounts: {
+					[entitlementKey]: [
+						{
+							model: "gpt-5.3-codex",
+							blockedUntil: 200,
+							reason: "plan-entitlement",
+							updatedAt: 100,
+						},
+					],
+				},
+			},
+			now: 100,
+		});
+
+		expect(matrix.entries[0]).toMatchObject({
+			available: false,
+			entitlementBlocked: true,
+			entitlementReason: "plan-entitlement",
+			entitlementWaitMs: 100,
+		});
+		expect(matrix.entries[0]?.reasons).toContain("account disabled");
+		expect(matrix.entries[0]?.reasons).toContain(
+			"entitlement blocked: plan-entitlement",
+		);
+	});
 });
 

test/test-model-matrix-script.test.ts

@@ -150,7 +150,33 @@ describe("test-model-matrix script helpers", () => {
 		);
 	});
 
-	it("downgrades unsupported smoke failures to skipped cases", async () => {
+	it("does not treat echoed prompt tokens as success after a terminal JSON failure", async () => {
+		const mod = await import("../scripts/test-model-matrix.js");
+		expect(
+			mod.__finalizeModelCaseResultForTests(
+				{ model: "gpt-5.2" },
+				0,
+				[
+					'{"type":"thread.started"}',
+					'{"type":"turn.started"}',
+					"MODEL_MATRIX_OK_8",
+					'{"type":"turn.failed","error":{"message":"model execution failed"}}',
+				].join("\n"),
+				"MODEL_MATRIX_OK_8",
+				false,
+				true,
+			),
+		).toEqual(
+			expect.objectContaining({
+				ok: false,
+				hasToken: true,
+				completed: false,
+				skipped: false,
+			}),
+		);
+	});
+
+	it("downgrades unsupported capability failures to skipped cases", async () => {
 		const mod = await import("../scripts/test-model-matrix.js");
 		expect(
 			mod.__finalizeModelCaseResultForTests(
@@ -168,6 +194,21 @@ describe("test-model-matrix script helpers", () => {
 			}),
 		);
 
+		expect(
+			mod.__finalizeModelCaseResultForTests(
+				{ model: "gpt-5.5-pro" },
+				1,
+				"{\"type\":\"turn.failed\",\"error\":{\"message\":\"The 'gpt-5.5-pro' model is not supported when using Codex with a ChatGPT account.\"}}",
+				"MODEL_MATRIX_OK_12",
+			),
+		).toEqual(
+			expect.objectContaining({
+				ok: false,
+				skipped: true,
+				skipReason: "unsupported-model",
+			}),
+		);
+
 		expect(
 			mod.__finalizeModelCaseResultForTests(
 				{ model: "gpt-5.2" },
@@ -185,6 +226,42 @@ describe("test-model-matrix script helpers", () => {
 		);
 	});
 
+	it("keeps strict capability and full timeout failures red", async () => {
+		const mod = await import("../scripts/test-model-matrix.js");
+
+		expect(
+			mod.__finalizeModelCaseResultForTests(
+				{ model: "gpt-5.5-pro" },
+				1,
+				"{\"type\":\"turn.failed\",\"error\":{\"message\":\"The 'gpt-5.5-pro' model is not supported when using Codex with a ChatGPT account.\"}}",
+				"MODEL_MATRIX_OK_13",
+				false,
+				true,
+			),
+		).toEqual(
+			expect.objectContaining({
+				ok: false,
+				skipped: false,
+				skipReason: null,
+			}),
+		);
+
+		expect(
+			mod.__finalizeModelCaseResultForTests(
+				{ model: "gpt-5.2" },
+				124,
+				"Timed out after 120000ms",
+				"MODEL_MATRIX_OK_14",
+			),
+		).toEqual(
+			expect.objectContaining({
+				ok: false,
+				skipped: false,
+				skipReason: null,
+			}),
+		);
+	});
+
 	it("filters non-path where output on Windows", async () => {
 		const platformSpy = vi
 			.spyOn(process, "platform", "get")