feat(test): add "test diff <run-a> <run-b>" to isolate run-to-run regressions (#168)

Andy00L · web-flow · commit 9457583f27b8 · 2026-07-05T12:33:26.000-07:00
* feat(test): add "test diff &lt;run-a&gt; &lt;run-b&gt;" to isolate run-to-run regressions

* test(diff): cover runDiff --dry-run branch (offline canned sample)
diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts
@@ -27,6 +27,7 @@ import {
   runCreateBatch,
   runCreateFromPlan,
   runDelete,
+  runDiff,
   runFailureGet,
   runFailureSummary,
   runGet,
@@ -121,6 +122,7 @@ describe('createTestCommand — surface', () => {
       'create-batch',
       'delete',
       'delete-batch',
+      'diff',
       'failure',
       'get',
       'list',
@@ -2833,6 +2835,131 @@ describe('runSteps', () => {
   });
 });
 
+describe('runDiff', () => {
+  const baseRun = {
+    testId: 'test_fe',
+    projectId: 'project_alice',
+    userId: 'u1',
+    source: 'cli',
+    createdAt: '2026-06-01T10:00:00.000Z',
+    startedAt: '2026-06-01T10:00:01.000Z',
+    finishedAt: '2026-06-01T10:00:30.000Z',
+    targetUrl: 'https://example.com',
+    createdFrom: null,
+    error: null,
+    videoUrl: null,
+    stepSummary: { total: 2, completed: 2, passedCount: 2, failedCount: 0 },
+  };
+  const makeStep = (index: string, status: string, error: string | null = null) => ({
+    stepIndex: index,
+    type: 'action',
+    action: `step ${index}`,
+    status,
+    description: `Step ${index}`,
+    error,
+    screenshotUrl: null,
+    htmlSnapshotUrl: null,
+    createdAt: '2026-06-01T10:00:05.000Z',
+  });
+  const RUN_GREEN = {
+    ...baseRun,
+    runId: 'run_green',
+    status: 'passed',
+    codeVersion: 'v1',
+    failedStepIndex: null,
+    failureKind: null,
+    steps: [makeStep('0001', 'passed'), makeStep('0002', 'passed')],
+  };
+  const RUN_RED = {
+    ...baseRun,
+    runId: 'run_red',
+    status: 'failed',
+    codeVersion: 'v2',
+    failedStepIndex: 2,
+    failureKind: 'assertion',
+    steps: [makeStep('0001', 'passed'), makeStep('0002', 'failed', 'heading not visible')],
+  };
+  const fetchForRuns = () =>
+    makeFetch(url => ({ body: url.includes('run_green') ? RUN_GREEN : RUN_RED }));
+
+  it('reports the verdict flip, the changed step with its error, and code drift, then exits 1', async () => {
+    const { credentialsPath } = makeCreds();
+    const out: string[] = [];
+    const rejection = await runDiff(
+      { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_red' },
+      { credentialsPath, fetchImpl: fetchForRuns(), stdout: line => out.push(line) },
+    ).catch((error: unknown) => error);
+    expect(rejection).toMatchObject({ exitCode: 1 });
+    const printed = JSON.parse(out.join('')) as {
+      verdictChanged: boolean;
+      codeVersionChanged: boolean;
+      crossTest: boolean;
+      changedSteps: Array<{ stepIndex: number; statusA: string; statusB: string; errorB?: string }>;
+    };
+    expect(printed.verdictChanged).toBe(true);
+    expect(printed.codeVersionChanged).toBe(true);
+    expect(printed.crossTest).toBe(false);
+    expect(printed.changedSteps).toHaveLength(1);
+    expect(printed.changedSteps[0]).toMatchObject({
+      stepIndex: 2,
+      statusA: 'passed',
+      statusB: 'failed',
+      errorB: 'heading not visible',
+    });
+  });
+
+  it('identical verdicts resolve with exit 0 and no step changes', async () => {
+    const { credentialsPath } = makeCreds();
+    const fetchImpl = makeFetch(() => ({ body: RUN_GREEN }));
+    const diff = await runDiff(
+      { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_green' },
+      { credentialsPath, fetchImpl, stdout: () => undefined },
+    );
+    expect(diff.verdictChanged).toBe(false);
+    expect(diff.changedSteps).toHaveLength(0);
+  });
+
+  it('warns (not fails) when the runs belong to different tests', async () => {
+    const { credentialsPath } = makeCreds();
+    const OTHER = { ...RUN_GREEN, runId: 'run_red', testId: 'test_other' };
+    const fetchImpl = makeFetch(url => ({
+      body: url.includes('run_green') ? RUN_GREEN : OTHER,
+    }));
+    const errs: string[] = [];
+    const diff = await runDiff(
+      { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_red' },
+      { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => errs.push(line) },
+    );
+    expect(diff.crossTest).toBe(true);
+    expect(errs.join('\n')).toContain('different tests');
+  });
+
+  it('--dry-run returns the canned sample fully offline (no credentials, no fetch)', async () => {
+    // Dry-run must not require credentials or hit the network — it returns a
+    // canned CliRunDiff so `--dry-run` shows the shape offline.
+    const diff = await runDiff(
+      {
+        profile: 'default',
+        output: 'json',
+        debug: false,
+        dryRun: true,
+        runA: 'run_aaa',
+        runB: 'run_bbb',
+      },
+      { stdout: () => undefined, stderr: () => undefined },
+    );
+    expect(diff.runA.runId).toBe('run_aaa');
+    expect(diff.runB.runId).toBe('run_bbb');
+    expect(diff.verdictChanged).toBe(true);
+    expect(diff.changedSteps).toHaveLength(1);
+    expect(diff.changedSteps[0]).toMatchObject({
+      stepIndex: 2,
+      statusA: 'passed',
+      statusB: 'failed',
+    });
+  });
+});
+
 describe('runResult', () => {
   it('JSON mode prints the §6.5 LatestResult shape verbatim', async () => {
     const { credentialsPath } = makeCreds();
diff --git a/src/commands/test.ts b/src/commands/test.ts
@@ -3664,6 +3664,196 @@ function mapRunStepToCliTestStep(step: RunStepDto, run: RunResponse): CliTestSte
   };
 }
 
+export interface DiffOptions extends CommonOptions {
+  runA: string;
+  runB: string;
+}
+
+/** One step whose status flipped between the two compared runs. */
+export interface CliDiffStep {
+  stepIndex: number;
+  statusA: string;
+  statusB: string;
+  /** First divergent failing side's error text, when the wire carried one. */
+  errorA?: string | null;
+  errorB?: string | null;
+}
+
+export interface CliRunDiff {
+  runA: {
+    runId: string;
+    testId: string;
+    status: string;
+    failureKind: string | null;
+    failedStepIndex: number | null;
+    codeVersion: string | null;
+  };
+  runB: {
+    runId: string;
+    testId: string;
+    status: string;
+    failureKind: string | null;
+    failedStepIndex: number | null;
+    codeVersion: string | null;
+  };
+  verdictChanged: boolean;
+  failedStepIndexChanged: boolean;
+  failureKindChanged: boolean;
+  codeVersionChanged: boolean;
+  /** True when the two runs belong to DIFFERENT tests (deltas may be meaningless). */
+  crossTest: boolean;
+  changedSteps: CliDiffStep[];
+}
+
+/**
+ * `test diff <runA> <runB>` (issue #124): isolate what regressed between two
+ * runs, the first question when CI goes red ("what changed since the last
+ * green run?"). Pure client-side composition of the existing per-run read
+ * (`GET /runs/{id}?includeSteps=true`); the endpoint accepts any two run-ids,
+ * so a cross-test pair is a WARNING, not an error. Exit 0 when the verdicts
+ * match, exit 1 when they differ, so the command is CI-scriptable.
+ */
+export async function runDiff(opts: DiffOptions, deps: TestDeps = {}): Promise<CliRunDiff> {
+  const out = makeOutput(opts.output, deps);
+  const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`));
+
+  if (opts.dryRun) {
+    emitDryRunBanner(stderrFn);
+    const sample: CliRunDiff = {
+      runA: {
+        runId: opts.runA,
+        testId: 'test_dryrun',
+        status: 'passed',
+        failureKind: null,
+        failedStepIndex: null,
+        codeVersion: 'v1',
+      },
+      runB: {
+        runId: opts.runB,
+        testId: 'test_dryrun',
+        status: 'failed',
+        failureKind: 'assertion',
+        failedStepIndex: 2,
+        codeVersion: 'v1',
+      },
+      verdictChanged: true,
+      failedStepIndexChanged: true,
+      failureKindChanged: true,
+      codeVersionChanged: false,
+      crossTest: false,
+      changedSteps: [{ stepIndex: 2, statusA: 'passed', statusB: 'failed' }],
+    };
+    out.print(sample, () => renderRunDiffText(sample));
+    return sample;
+  }
+
+  const client = makeClient(opts, deps);
+  const [runA, runB] = await Promise.all([
+    client.getRun(opts.runA, { includeSteps: true }),
+    client.getRun(opts.runB, { includeSteps: true }),
+  ]);
+
+  const crossTest = runA.testId !== runB.testId;
+  if (crossTest) {
+    stderrFn(
+      `⚠ the two runs belong to different tests (${runA.testId} vs ${runB.testId}) — step deltas may be meaningless`,
+    );
+  }
+
+  const stepsByIndex = (
+    run: RunResponse,
+  ): Map<number, { status: string; error: string | null }> => {
+    const map = new Map<number, { status: string; error: string | null }>();
+    for (const step of run.steps ?? []) {
+      const index = parseInt(step.stepIndex, 10);
+      if (Number.isInteger(index))
+        map.set(index, { status: step.status ?? 'unknown', error: step.error });
+    }
+    return map;
+  };
+  const stepsA = stepsByIndex(runA);
+  const stepsB = stepsByIndex(runB);
+  const allIndexes = [...new Set([...stepsA.keys(), ...stepsB.keys()])].sort(
+    (left, right) => left - right,
+  );
+  const changedSteps: CliDiffStep[] = [];
+  for (const index of allIndexes) {
+    const sideA = stepsA.get(index);
+    const sideB = stepsB.get(index);
+    const statusA = sideA?.status ?? 'absent';
+    const statusB = sideB?.status ?? 'absent';
+    if (statusA === statusB) continue;
+    changedSteps.push({
+      stepIndex: index,
+      statusA,
+      statusB,
+      ...(sideA?.error ? { errorA: sideA.error } : {}),
+      ...(sideB?.error ? { errorB: sideB.error } : {}),
+    });
+  }
+
+  const summarize = (run: RunResponse) => ({
+    runId: run.runId,
+    testId: run.testId,
+    status: run.status,
+    failureKind: run.failureKind ?? null,
+    failedStepIndex: run.failedStepIndex,
+    codeVersion: run.codeVersion ?? null,
+  });
+  const diff: CliRunDiff = {
+    runA: summarize(runA),
+    runB: summarize(runB),
+    verdictChanged: runA.status !== runB.status,
+    failedStepIndexChanged: runA.failedStepIndex !== runB.failedStepIndex,
+    failureKindChanged: (runA.failureKind ?? null) !== (runB.failureKind ?? null),
+    codeVersionChanged: (runA.codeVersion ?? null) !== (runB.codeVersion ?? null),
+    crossTest,
+    changedSteps,
+  };
+  out.print(diff, () => renderRunDiffText(diff));
+
+  if (diff.verdictChanged) {
+    // Result already printed; the typed exit makes `test diff` a CI gate.
+    throw new CLIError(
+      `verdicts differ: ${runA.runId}=${runA.status} vs ${runB.runId}=${runB.status}`,
+      1,
+    );
+  }
+  return diff;
+}
+
+function renderRunDiffText(diff: CliRunDiff): string {
+  const lines: string[] = [];
+  lines.push(`runA:  ${diff.runA.runId}  ${diff.runA.status}  (test ${diff.runA.testId})`);
+  lines.push(`runB:  ${diff.runB.runId}  ${diff.runB.status}  (test ${diff.runB.testId})`);
+  lines.push(
+    `verdict:          ${diff.verdictChanged ? `${diff.runA.status} -> ${diff.runB.status}` : `unchanged (${diff.runA.status})`}`,
+  );
+  if (diff.failureKindChanged)
+    lines.push(
+      `failureKind:      ${diff.runA.failureKind ?? '(none)'} -> ${diff.runB.failureKind ?? '(none)'}`,
+    );
+  if (diff.failedStepIndexChanged)
+    lines.push(
+      `failedStepIndex:  ${diff.runA.failedStepIndex ?? '(none)'} -> ${diff.runB.failedStepIndex ?? '(none)'}`,
+    );
+  lines.push(
+    `codeVersion:      ${diff.codeVersionChanged ? `${diff.runA.codeVersion ?? '(none)'} -> ${diff.runB.codeVersion ?? '(none)'} (code drift)` : 'unchanged'}`,
+  );
+  if (diff.changedSteps.length === 0) {
+    lines.push('steps:            no per-step status changes');
+  } else {
+    lines.push(`steps changed:    ${diff.changedSteps.length}`);
+    for (const step of diff.changedSteps) {
+      lines.push(`  #${step.stepIndex}  ${step.statusA} -> ${step.statusB}`);
+      if (step.errorB) lines.push(`      error(B): ${step.errorB.replace(/\s+/g, ' ').trim()}`);
+      else if (step.errorA)
+        lines.push(`      error(A): ${step.errorA.replace(/\s+/g, ' ').trim()}`);
+    }
+  }
+  return lines.join('\n');
+}
+
 export async function runSteps(
   opts: StepsOptions,
   deps: TestDeps = {},
@@ -7413,6 +7603,16 @@ export function createTestCommand(deps: TestDeps = {}): Command {
       );
     });
 
+  test
+    .command('diff <run-a> <run-b>')
+    .description(
+      'Compare two runs and print what regressed: verdict, failureKind, failedStepIndex, per-step status flips, codeVersion drift. Exit 0 when verdicts match, 1 when they differ.',
+    )
+    .addHelpText('after', GLOBAL_OPTS_HINT)
+    .action(async (runA: string, runB: string, _cmdOpts: unknown, command: Command) => {
+      await runDiff({ ...resolveCommonOptions(command), runA, runB }, deps);
+    });
+
   test
     .command('result <test-id>')
     .description(
diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap
@@ -200,6 +200,11 @@ Commands:
   steps [options] <test-id>             List the steps for a test (server
                                         returns the cumulative log across every
                                         run; use --run-id to scope to one run)
+  diff <run-a> <run-b>                  Compare two runs and print what
+                                        regressed: verdict, failureKind,
+                                        failedStepIndex, per-step status flips,
+                                        codeVersion drift. Exit 0 when verdicts
+                                        match, 1 when they differ.
   result [options] <test-id>            Get the latest result for a test (default) or list prior runs (--history).
   
   --output json shape differs by mode:
