fix(benchmarks): Run all Claude UI directory suites

Continue running Claude UI benchmark suites after an earlier suite fails, while preserving a non-zero final exit code.

Extract the directory runner into a testable module and cover the failure behavior.

Author: cameroncooke

benchmarks/claude-ui/run-directory.ts

@@ -1,53 +1,7 @@
 #!/usr/bin/env tsx
-import { access, readdir } from 'node:fs/promises';
-import path from 'node:path';
-import { main } from '../../src/benchmarks/claude-ui/harness.ts';
+import { runDirectory } from '../../src/benchmarks/claude-ui/run-directory.ts';
 
-async function directoryExists(directory: string): Promise<boolean> {
-  try {
-    await access(directory);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-async function suitePaths(directory: string): Promise<string[]> {
-  if (!(await directoryExists(directory))) return [];
-  const entries = await readdir(directory, { withFileTypes: true });
-  return entries
-    .filter(
-      (entry) => entry.isFile() && (entry.name.endsWith('.yml') || entry.name.endsWith('.yaml')),
-    )
-    .map((entry) => path.join(directory, entry.name))
-    .sort();
-}
-
-async function run(): Promise<number> {
-  const directory = process.argv[2];
-  const maybeLabel = process.argv[3];
-  const label = maybeLabel && !maybeLabel.startsWith('-') ? maybeLabel : directory;
-  const forwardedArgs =
-    maybeLabel && !maybeLabel.startsWith('-') ? process.argv.slice(4) : process.argv.slice(3);
-  if (!directory) {
-    console.error('Usage: run-directory.ts <suite-directory> [label] [benchmark args...]');
-    return 1;
-  }
-
-  const suites = await suitePaths(directory);
-  if (suites.length === 0) {
-    console.error(`No ${label} Claude UI benchmark suites found in ${directory}`);
-    return 1;
-  }
-
-  for (const suite of suites) {
-    const exitCode = await main(['--suite', suite, ...forwardedArgs]);
-    if (exitCode !== 0) return exitCode;
-  }
-  return 0;
-}
-
-run()
+runDirectory(process.argv.slice(2))
   .then((exitCode) => {
     process.exitCode = exitCode;
   })

src/benchmarks/claude-ui/__tests__/run-directory.test.ts

@@ -0,0 +1,34 @@
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { runDirectory } from '../run-directory.ts';
+
+describe('runDirectory', () => {
+  it('continues running later suites after an earlier suite returns non-zero', async () => {
+    const directory = await mkdtemp(path.join(os.tmpdir(), 'xcodebuildmcp-bench-suites-'));
+    try {
+      await writeFile(path.join(directory, 'a.yml'), 'name: a\n', 'utf8');
+      await writeFile(path.join(directory, 'b.yml'), 'name: b\n', 'utf8');
+      await writeFile(path.join(directory, 'c.yml'), 'name: c\n', 'utf8');
+
+      const calls: string[][] = [];
+      const exitCode = await runDirectory(
+        [directory, 'private', '--model', 'opus'],
+        async (args) => {
+          calls.push(args);
+          return args[1]?.endsWith('b.yml') ? 1 : 0;
+        },
+      );
+
+      expect(exitCode).toBe(1);
+      expect(calls).toEqual([
+        ['--suite', path.join(directory, 'a.yml'), '--model', 'opus'],
+        ['--suite', path.join(directory, 'b.yml'), '--model', 'opus'],
+        ['--suite', path.join(directory, 'c.yml'), '--model', 'opus'],
+      ]);
+    } finally {
+      await rm(directory, { recursive: true, force: true });
+    }
+  });
+});

src/benchmarks/claude-ui/run-directory.ts

@@ -0,0 +1,51 @@
+import { access, readdir } from 'node:fs/promises';
+import path from 'node:path';
+import { main } from './harness.ts';
+
+async function directoryExists(directory: string): Promise<boolean> {
+  try {
+    await access(directory);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export async function suitePaths(directory: string): Promise<string[]> {
+  if (!(await directoryExists(directory))) return [];
+  const entries = await readdir(directory, { withFileTypes: true });
+  return entries
+    .filter(
+      (entry) => entry.isFile() && (entry.name.endsWith('.yml') || entry.name.endsWith('.yaml')),
+    )
+    .map((entry) => path.join(directory, entry.name))
+    .sort();
+}
+
+type RunSuite = (args: string[]) => Promise<number>;
+
+export async function runDirectory(args: string[], runSuite: RunSuite = main): Promise<number> {
+  const directory = args[0];
+  const maybeLabel = args[1];
+  const label = maybeLabel && !maybeLabel.startsWith('-') ? maybeLabel : directory;
+  const forwardedArgs = maybeLabel && !maybeLabel.startsWith('-') ? args.slice(2) : args.slice(1);
+  if (!directory) {
+    console.error('Usage: run-directory.ts <suite-directory> [label] [benchmark args...]');
+    return 1;
+  }
+
+  const suites = await suitePaths(directory);
+  if (suites.length === 0) {
+    console.error(`No ${label} Claude UI benchmark suites found in ${directory}`);
+    return 1;
+  }
+
+  let finalExitCode = 0;
+  for (const suite of suites) {
+    const exitCode = await runSuite(['--suite', suite, ...forwardedArgs]);
+    if (exitCode !== 0 && finalExitCode === 0) {
+      finalExitCode = exitCode;
+    }
+  }
+  return finalExitCode;
+}