feat: add --spec-version option to tier-check

Allows filtering conformance scenarios by target spec version, so SDKs
are only scored against scenarios applicable to the version they target.
This prevents legacy backward-compatibility scenarios from counting
against SDKs that don't claim legacy support.

Author: DePasqualeOrg

src/tier-check/checks/test-conformance-results.ts

@@ -6,6 +6,8 @@ import { ConformanceResult } from '../types';
 import {
   listScenarios,
   listActiveClientScenarios,
+  listScenariosForSpec,
+  listClientScenariosForSpec,
   getScenarioSpecVersions
 } from '../../scenarios';
 import { ConformanceCheck, SpecVersion } from '../../types';
@@ -166,6 +168,7 @@ function reconcileWithExpected(
 export async function checkConformance(options: {
   serverUrl?: string;
   skip?: boolean;
+  specVersion?: SpecVersion;
 }): Promise<ConformanceResult> {
   if (options.skip || !options.serverUrl) {
     return {
@@ -179,10 +182,13 @@ export async function checkConformance(options: {
   }
 
   const outputDir = mkdtempSync(join(tmpdir(), 'tier-check-server-'));
+  const specVersionArg = options.specVersion
+    ? `--spec-version ${options.specVersion}`
+    : '';
 
   try {
     execSync(
-      `node dist/index.js server --url ${options.serverUrl} -o ${outputDir}`,
+      `node dist/index.js server --url ${options.serverUrl} -o ${outputDir} ${specVersionArg}`,
       {
         cwd: process.cwd(),
         stdio: ['pipe', 'pipe', 'pipe'],
@@ -193,9 +199,16 @@ export async function checkConformance(options: {
     // Non-zero exit is expected when tests fail — results are still in outputDir
   }
 
+  const activeScenarios = new Set(listActiveClientScenarios());
+  const expectedScenarios = options.specVersion
+    ? listClientScenariosForSpec(options.specVersion).filter((s) =>
+        activeScenarios.has(s)
+      )
+    : [...activeScenarios];
+
   return reconcileWithExpected(
     parseOutputDir(outputDir),
-    listActiveClientScenarios(),
+    expectedScenarios,
     'server'
   );
 }
@@ -206,6 +219,7 @@ export async function checkConformance(options: {
 export async function checkClientConformance(options: {
   clientCmd?: string;
   skip?: boolean;
+  specVersion?: SpecVersion;
 }): Promise<ConformanceResult> {
   if (options.skip || !options.clientCmd) {
     return {
@@ -219,10 +233,13 @@ export async function checkClientConformance(options: {
   }
 
   const outputDir = mkdtempSync(join(tmpdir(), 'tier-check-client-'));
+  const specVersionArg = options.specVersion
+    ? `--spec-version ${options.specVersion}`
+    : '';
 
   try {
     execSync(
-      `node dist/index.js client --command '${options.clientCmd}' --suite all -o ${outputDir}`,
+      `node dist/index.js client --command '${options.clientCmd}' --suite all -o ${outputDir} ${specVersionArg}`,
       {
         cwd: process.cwd(),
         stdio: ['pipe', 'pipe', 'pipe'],
@@ -233,5 +250,9 @@ export async function checkClientConformance(options: {
     // Non-zero exit is expected when tests fail — results are still in outputDir
   }
 
-  return reconcileWithExpected(parseOutputDir(outputDir), listScenarios());
+  const expectedScenarios = options.specVersion
+    ? listScenariosForSpec(options.specVersion)
+    : listScenarios();
+
+  return reconcileWithExpected(parseOutputDir(outputDir), expectedScenarios);
 }

src/tier-check/index.ts

@@ -13,6 +13,8 @@ import { checkSpecTracking } from './checks/spec-tracking';
 import { computeTier } from './tier-logic';
 import { formatJson, formatMarkdown, formatTerminal } from './output';
 import { TierScorecard } from './types';
+import { ALL_SPEC_VERSIONS } from '../scenarios';
+import { SpecVersion } from '../types';
 
 function parseRepo(repo: string): { owner: string; repo: string } {
   const parts = repo.split('/');
@@ -48,10 +50,24 @@ export function createTierCheckCommand(): Command {
       '--token <token>',
       'GitHub token (defaults to GITHUB_TOKEN env var)'
     )
+    .option(
+      '--spec-version <version>',
+      'Only run conformance scenarios for this spec version'
+    )
     .action(async (options) => {
       const { owner, repo } = parseRepo(options.repo);
       let token = options.token || process.env.GITHUB_TOKEN;
 
+      let specVersion: SpecVersion | undefined;
+      if (options.specVersion) {
+        if (!ALL_SPEC_VERSIONS.includes(options.specVersion as SpecVersion)) {
+          console.error(`Unknown spec version: ${options.specVersion}`);
+          console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`);
+          process.exit(1);
+        }
+        specVersion = options.specVersion as SpecVersion;
+      }
+
       if (!token) {
         // Try to get token from GitHub CLI
         try {
@@ -90,14 +106,16 @@ export function createTierCheckCommand(): Command {
       ] = await Promise.all([
         checkConformance({
           serverUrl: options.conformanceServerUrl,
-          skip: options.skipConformance
+          skip: options.skipConformance,
+          specVersion
         }).then((r) => {
           console.error('  ✓ Server Conformance');
           return r;
         }),
         checkClientConformance({
           clientCmd: options.clientCmd,
-          skip: options.skipConformance || !options.clientCmd
+          skip: options.skipConformance || !options.clientCmd,
+          specVersion
         }).then((r) => {
           console.error('  ✓ Client Conformance');
           return r;