feat: add --spec-version option to tier-check (#176)

* feat: add --spec-version option to tier-check

* refactor: share resolveSpecVersion between CLI and tier-check

Move resolveSpecVersion to src/scenarios/index.ts so tier-check can
reuse it instead of duplicating the validation logic.

---------

Co-authored-by: Felix Weinberger <fweinberger@anthropic.com>

Author: DePasqualeOrg

src/index.ts

@@ -24,7 +24,7 @@ import {
   listScenariosForSpec,
   listClientScenariosForSpec,
   getScenarioSpecVersions,
-  ALL_SPEC_VERSIONS
+  resolveSpecVersion
 } from './scenarios';
 import type { SpecVersion } from './scenarios';
 import { ConformanceCheck } from './types';
@@ -37,15 +37,6 @@ import {
 import { createTierCheckCommand } from './tier-check';
 import packageJson from '../package.json';
 
-function resolveSpecVersion(value: string): SpecVersion {
-  if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) {
-    return value as SpecVersion;
-  }
-  console.error(`Unknown spec version: ${value}`);
-  console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`);
-  process.exit(1);
-}
-
 // Note on naming: `command` refers to which CLI command is calling this.
 // The `client` command tests Scenario objects (which test clients),
 // and the `server` command tests ClientScenario objects (which test servers).

src/scenarios/index.ts

@@ -227,6 +227,15 @@ export const ALL_SPEC_VERSIONS: SpecVersion[] = [
   'extension'
 ];
 
+export function resolveSpecVersion(value: string): SpecVersion {
+  if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) {
+    return value as SpecVersion;
+  }
+  console.error(`Unknown spec version: ${value}`);
+  console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`);
+  process.exit(1);
+}
+
 export function listScenariosForSpec(version: SpecVersion): string[] {
   return scenariosList
     .filter((s) => s.specVersions.includes(version))

src/tier-check/checks/test-conformance-results.ts

@@ -6,6 +6,8 @@ import { ConformanceResult } from '../types';
 import {
   listScenarios,
   listActiveClientScenarios,
+  listScenariosForSpec,
+  listClientScenariosForSpec,
   getScenarioSpecVersions
 } from '../../scenarios';
 import { ConformanceCheck, SpecVersion } from '../../types';
@@ -166,6 +168,7 @@ function reconcileWithExpected(
 export async function checkConformance(options: {
   serverUrl?: string;
   skip?: boolean;
+  specVersion?: SpecVersion;
 }): Promise<ConformanceResult> {
   if (options.skip || !options.serverUrl) {
     return {
@@ -179,23 +182,37 @@ export async function checkConformance(options: {
   }
 
   const outputDir = mkdtempSync(join(tmpdir(), 'tier-check-server-'));
+  const args = [
+    process.argv[1],
+    'server',
+    '--url',
+    options.serverUrl,
+    '-o',
+    outputDir
+  ];
+  if (options.specVersion) {
+    args.push('--spec-version', options.specVersion);
+  }
 
   try {
-    execFileSync(
-      process.execPath,
-      [process.argv[1], 'server', '--url', options.serverUrl, '-o', outputDir],
-      {
-        stdio: ['pipe', 'pipe', 'pipe'],
-        timeout: 120_000
-      }
-    );
+    execFileSync(process.execPath, args, {
+      stdio: ['pipe', 'pipe', 'pipe'],
+      timeout: 120_000
+    });
   } catch {
     // Non-zero exit is expected when tests fail — results are still in outputDir
   }
 
+  const activeScenarios = new Set(listActiveClientScenarios());
+  const expectedScenarios = options.specVersion
+    ? listClientScenariosForSpec(options.specVersion).filter((s) =>
+        activeScenarios.has(s)
+      )
+    : [...activeScenarios];
+
   return reconcileWithExpected(
     parseOutputDir(outputDir),
-    listActiveClientScenarios(),
+    expectedScenarios,
     'server'
   );
 }
@@ -206,6 +223,7 @@ export async function checkConformance(options: {
 export async function checkClientConformance(options: {
   clientCmd?: string;
   skip?: boolean;
+  specVersion?: SpecVersion;
 }): Promise<ConformanceResult> {
   if (options.skip || !options.clientCmd) {
     return {
@@ -219,28 +237,32 @@ export async function checkClientConformance(options: {
   }
 
   const outputDir = mkdtempSync(join(tmpdir(), 'tier-check-client-'));
+  const args = [
+    process.argv[1],
+    'client',
+    '--command',
+    options.clientCmd,
+    '--suite',
+    'all',
+    '-o',
+    outputDir
+  ];
+  if (options.specVersion) {
+    args.push('--spec-version', options.specVersion);
+  }
 
   try {
-    execFileSync(
-      process.execPath,
-      [
-        process.argv[1],
-        'client',
-        '--command',
-        options.clientCmd,
-        '--suite',
-        'all',
-        '-o',
-        outputDir
-      ],
-      {
-        stdio: ['pipe', 'pipe', 'pipe'],
-        timeout: 120_000
-      }
-    );
+    execFileSync(process.execPath, args, {
+      stdio: ['pipe', 'pipe', 'pipe'],
+      timeout: 120_000
+    });
   } catch {
     // Non-zero exit is expected when tests fail — results are still in outputDir
   }
 
-  return reconcileWithExpected(parseOutputDir(outputDir), listScenarios());
+  const expectedScenarios = options.specVersion
+    ? listScenariosForSpec(options.specVersion)
+    : listScenarios();
+
+  return reconcileWithExpected(parseOutputDir(outputDir), expectedScenarios);
 }

src/tier-check/index.ts

@@ -13,6 +13,7 @@ import { checkSpecTracking } from './checks/spec-tracking';
 import { computeTier } from './tier-logic';
 import { formatJson, formatMarkdown, formatTerminal } from './output';
 import { TierScorecard } from './types';
+import { resolveSpecVersion } from '../scenarios';
 
 function parseRepo(repo: string): { owner: string; repo: string } {
   const parts = repo.split('/');
@@ -48,10 +49,18 @@ export function createTierCheckCommand(): Command {
       '--token <token>',
       'GitHub token (defaults to GITHUB_TOKEN env var)'
     )
+    .option(
+      '--spec-version <version>',
+      'Only run conformance scenarios for this spec version'
+    )
     .action(async (options) => {
       const { owner, repo } = parseRepo(options.repo);
       let token = options.token || process.env.GITHUB_TOKEN;
 
+      const specVersion = options.specVersion
+        ? resolveSpecVersion(options.specVersion)
+        : undefined;
+
       if (!token) {
         // Try to get token from GitHub CLI
         try {
@@ -90,14 +99,16 @@ export function createTierCheckCommand(): Command {
       ] = await Promise.all([
         checkConformance({
           serverUrl: options.conformanceServerUrl,
-          skip: options.skipConformance
+          skip: options.skipConformance,
+          specVersion
         }).then((r) => {
           console.error('  ✓ Server Conformance');
           return r;
         }),
         checkClientConformance({
           clientCmd: options.clientCmd,
-          skip: options.skipConformance || !options.clientCmd
+          skip: options.skipConformance || !options.clientCmd,
+          specVersion
         }).then((r) => {
           console.error('  ✓ Client Conformance');
           return r;