feat: make results directory output optional (#107)

Add -o, --output-dir option to both client and server commands.
By default, no results directory is created - only console output.
When -o is specified, results are saved to that directory with
timestamped subdirectories.

This eliminates the need to add 'results/' to .gitignore in every
repo where conformance tests are run.

Claude-Generated-By: Claude Code (cli/claude-opus-4-5=100%)
Claude-Steers: 0
Claude-Permission-Prompts: 2
Claude-Escapes: 0

Author: pcarleton

src/index.ts

@@ -39,11 +39,13 @@ program
   .option('--scenario <scenario>', 'Scenario to test')
   .option('--suite <suite>', 'Run a suite of tests in parallel (e.g., "auth")')
   .option('--timeout <ms>', 'Timeout in milliseconds', '30000')
+  .option('-o, --output-dir <path>', 'Save results to this directory')
   .option('--verbose', 'Show verbose output')
   .action(async (options) => {
     try {
       const timeout = parseInt(options.timeout, 10);
       const verbose = options.verbose ?? false;
+      const outputDir = options.outputDir;
 
       // Handle suite mode
       if (options.suite) {
@@ -78,7 +80,8 @@ program
               const result = await runConformanceTest(
                 options.command,
                 scenarioName,
-                timeout
+                timeout,
+                outputDir
               );
               return {
                 scenario: scenarioName,
@@ -163,15 +166,16 @@ program
 
       // If no command provided, run in interactive mode
       if (!validated.command) {
-        await runInteractiveMode(validated.scenario, verbose);
+        await runInteractiveMode(validated.scenario, verbose, outputDir);
         process.exit(0);
       }
 
       // Otherwise run conformance test
       const result = await runConformanceTest(
         validated.command,
         validated.scenario,
-        timeout
+        timeout,
+        outputDir
       );
 
       const { overallFailure } = printClientResults(
@@ -209,19 +213,22 @@ program
     'Suite to run: "active" (default, excludes pending), "all", or "pending"',
     'active'
   )
+  .option('-o, --output-dir <path>', 'Save results to this directory')
   .option('--verbose', 'Show verbose output (JSON instead of pretty print)')
   .action(async (options) => {
     try {
       // Validate options with Zod
       const validated = ServerOptionsSchema.parse(options);
 
       const verbose = options.verbose ?? false;
+      const outputDir = options.outputDir;
 
       // If a single scenario is specified, run just that one
       if (validated.scenario) {
         const result = await runServerConformanceTest(
           validated.url,
-          validated.scenario
+          validated.scenario,
+          outputDir
         );
 
         const { failed } = printServerResults(
@@ -259,7 +266,8 @@ program
           try {
             const result = await runServerConformanceTest(
               validated.url,
-              scenarioName
+              scenarioName,
+              outputDir
             );
             allResults.push({ scenario: scenarioName, checks: result.checks });
           } catch (error) {

src/runner/client.ts

@@ -3,7 +3,7 @@ import { promises as fs } from 'fs';
 import path from 'path';
 import { ConformanceCheck } from '../types';
 import { getScenario } from '../scenarios';
-import { ensureResultsDir, createResultDir, formatPrettyChecks } from './utils';
+import { createResultDir, formatPrettyChecks } from './utils';
 
 export interface ClientExecutionResult {
   exitCode: number;
@@ -91,15 +91,19 @@ async function executeClient(
 export async function runConformanceTest(
   clientCommand: string,
   scenarioName: string,
-  timeout: number = 30000
+  timeout: number = 30000,
+  outputDir?: string
 ): Promise<{
   checks: ConformanceCheck[];
   clientOutput: ClientExecutionResult;
-  resultDir: string;
+  resultDir?: string;
 }> {
-  await ensureResultsDir();
-  const resultDir = createResultDir(scenarioName);
-  await fs.mkdir(resultDir, { recursive: true });
+  let resultDir: string | undefined;
+
+  if (outputDir) {
+    resultDir = createResultDir(outputDir, scenarioName);
+    await fs.mkdir(resultDir, { recursive: true });
+  }
 
   // Scenario is guaranteed to exist by CLI validation
   const scenario = getScenario(scenarioName)!;
@@ -138,16 +142,24 @@ export async function runConformanceTest(
 
     const checks = scenario.getChecks();
 
-    await fs.writeFile(
-      path.join(resultDir, 'checks.json'),
-      JSON.stringify(checks, null, 2)
-    );
+    if (resultDir) {
+      await fs.writeFile(
+        path.join(resultDir, 'checks.json'),
+        JSON.stringify(checks, null, 2)
+      );
 
-    await fs.writeFile(path.join(resultDir, 'stdout.txt'), clientOutput.stdout);
+      await fs.writeFile(
+        path.join(resultDir, 'stdout.txt'),
+        clientOutput.stdout
+      );
 
-    await fs.writeFile(path.join(resultDir, 'stderr.txt'), clientOutput.stderr);
+      await fs.writeFile(
+        path.join(resultDir, 'stderr.txt'),
+        clientOutput.stderr
+      );
 
-    console.error(`Results saved to ${resultDir}`);
+      console.error(`Results saved to ${resultDir}`);
+    }
 
     return {
       checks,
@@ -244,11 +256,15 @@ export function printClientResults(
 
 export async function runInteractiveMode(
   scenarioName: string,
-  verbose: boolean = false
+  verbose: boolean = false,
+  outputDir?: string
 ): Promise<void> {
-  await ensureResultsDir();
-  const resultDir = createResultDir(scenarioName);
-  await fs.mkdir(resultDir, { recursive: true });
+  let resultDir: string | undefined;
+
+  if (outputDir) {
+    resultDir = createResultDir(outputDir, scenarioName);
+    await fs.mkdir(resultDir, { recursive: true });
+  }
 
   // Scenario is guaranteed to exist by CLI validation
   const scenario = getScenario(scenarioName)!;
@@ -257,23 +273,29 @@ export async function runInteractiveMode(
   const urls = await scenario.start();
 
   console.log(`Server URL: ${urls.serverUrl}`);
-  console.log('Press Ctrl+C to stop and save checks...');
+  console.log('Press Ctrl+C to stop...');
 
   const handleShutdown = async () => {
     console.log('\nShutting down...');
 
     const checks = scenario.getChecks();
-    await fs.writeFile(
-      path.join(resultDir, 'checks.json'),
-      JSON.stringify(checks, null, 2)
-    );
+
+    if (resultDir) {
+      await fs.writeFile(
+        path.join(resultDir, 'checks.json'),
+        JSON.stringify(checks, null, 2)
+      );
+    }
 
     if (verbose) {
       console.log(`\nChecks:\n${JSON.stringify(checks, null, 2)}`);
     } else {
       console.log(`\nChecks:\n${formatPrettyChecks(checks)}`);
     }
-    console.log(`\nChecks saved to ${resultDir}/checks.json`);
+
+    if (resultDir) {
+      console.log(`\nChecks saved to ${resultDir}/checks.json`);
+    }
 
     await scenario.stop();
     process.exit(0);

src/runner/index.ts

@@ -15,7 +15,6 @@ export {
 
 // Export utilities
 export {
-  ensureResultsDir,
   createResultDir,
   formatPrettyChecks,
   getStatusColor,

src/runner/server.ts

@@ -2,7 +2,7 @@ import { promises as fs } from 'fs';
 import path from 'path';
 import { ConformanceCheck } from '../types';
 import { getClientScenario } from '../scenarios';
-import { ensureResultsDir, createResultDir, formatPrettyChecks } from './utils';
+import { createResultDir, formatPrettyChecks } from './utils';
 
 /**
  * Format markdown-style text for terminal output using ANSI codes
@@ -19,15 +19,19 @@ function formatMarkdown(text: string): string {
 
 export async function runServerConformanceTest(
   serverUrl: string,
-  scenarioName: string
+  scenarioName: string,
+  outputDir?: string
 ): Promise<{
   checks: ConformanceCheck[];
-  resultDir: string;
+  resultDir?: string;
   scenarioDescription: string;
 }> {
-  await ensureResultsDir();
-  const resultDir = createResultDir(scenarioName, 'server');
-  await fs.mkdir(resultDir, { recursive: true });
+  let resultDir: string | undefined;
+
+  if (outputDir) {
+    resultDir = createResultDir(outputDir, scenarioName, 'server');
+    await fs.mkdir(resultDir, { recursive: true });
+  }
 
   // Scenario is guaranteed to exist by CLI validation
   const scenario = getClientScenario(scenarioName)!;
@@ -38,12 +42,14 @@ export async function runServerConformanceTest(
 
   const checks = await scenario.run(serverUrl);
 
-  await fs.writeFile(
-    path.join(resultDir, 'checks.json'),
-    JSON.stringify(checks, null, 2)
-  );
+  if (resultDir) {
+    await fs.writeFile(
+      path.join(resultDir, 'checks.json'),
+      JSON.stringify(checks, null, 2)
+    );
 
-  console.log(`Results saved to ${resultDir}`);
+    console.log(`Results saved to ${resultDir}`);
+  }
 
   return {
     checks,

src/runner/utils.ts

@@ -1,4 +1,3 @@
-import { promises as fs } from 'fs';
 import path from 'path';
 import { ConformanceCheck } from '../types';
 
@@ -51,14 +50,12 @@ export function formatPrettyChecks(checks: ConformanceCheck[]): string {
     .join('\n');
 }
 
-export async function ensureResultsDir(): Promise<string> {
-  const resultsDir = path.join(process.cwd(), 'results');
-  await fs.mkdir(resultsDir, { recursive: true });
-  return resultsDir;
-}
-
-export function createResultDir(scenario: string, prefix = ''): string {
+export function createResultDir(
+  baseDir: string,
+  scenario: string,
+  prefix = ''
+): string {
   const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
   const dirName = prefix ? `${prefix}-${scenario}` : scenario;
-  return path.join('results', `${dirName}-${timestamp}`);
+  return path.join(baseDir, `${dirName}-${timestamp}`);
 }