feat(test): add "test lint" offline plan/steps validator (#176)

* feat(test): add "test lint" offline plan/steps validator

* fix(lint): report physical JSONL line numbers (blank lines no longer shift them)

Author: Andy00L

src/commands/test.test.ts

@@ -31,6 +31,7 @@ import {
   runFailureGet,
   runFailureSummary,
   runGet,
+  runLint,
   runList,
   runPlanPut,
   runResult,
@@ -126,6 +127,7 @@ describe('createTestCommand — surface', () => {
       'failure',
       'flaky',
       'get',
+      'lint',
       'list',
       'plan',
       'rerun',
@@ -2961,6 +2963,85 @@ describe('runDiff', () => {
   });
 });
 
+describe('runLint', () => {
+  const VALID_PLAN = JSON.stringify({
+    projectId: 'project_alice',
+    type: 'frontend',
+    name: 'Checkout works',
+    planSteps: [
+      { type: 'action', description: 'Open the cart' },
+      { type: 'assertion', description: 'Assert the total is visible' },
+    ],
+  });
+  const INVALID_PLAN = JSON.stringify({
+    projectId: 'project_alice',
+    type: 'frontend',
+    name: 'Broken',
+    planSteps: [{ type: 'hover', description: 'Bad step type' }],
+  });
+
+  it('a directory with valid and invalid plans reports EVERY problem and exits 5', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'cli-lint-'));
+    writeFileSync(join(dir, 'a-valid.json'), VALID_PLAN, 'utf8');
+    writeFileSync(join(dir, 'b-invalid.json'), INVALID_PLAN, 'utf8');
+    writeFileSync(join(dir, 'c-notjson.json'), '{oops', 'utf8');
+    const out: string[] = [];
+    const rejection = await runLint(
+      { profile: 'default', output: 'json', debug: false, planFromDir: dir },
+      { stdout: line => out.push(line) },
+    ).catch((error: unknown) => error);
+    expect(rejection).toMatchObject({ exitCode: 5 });
+    const report = JSON.parse(out.join('')) as {
+      checked: number;
+      valid: number;
+      issues: Array<{ file: string; field: string }>;
+    };
+    // All three files were checked (no first-error-fatal bailout).
+    expect(report.checked).toBe(3);
+    expect(report.valid).toBe(1);
+    expect(report.issues.map(issue => issue.file).sort()).toEqual([
+      'b-invalid.json',
+      'c-notjson.json',
+    ]);
+  });
+
+  it('an all-valid directory resolves with exit 0 and no network/credentials needed', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'cli-lint-ok-'));
+    writeFileSync(join(dir, 'a.json'), VALID_PLAN, 'utf8');
+    const report = await runLint(
+      { profile: 'default', output: 'json', debug: false, planFromDir: dir },
+      { stdout: () => undefined },
+    );
+    expect(report).toMatchObject({ checked: 1, valid: 1, issues: [] });
+  });
+
+  it('a JSONL file reports each bad line with its PHYSICAL line number (blank lines skipped, not renumbered)', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'cli-lint-jsonl-'));
+    const file = join(dir, 'plans.jsonl');
+    // A blank separator line sits between entries: line 1 valid, line 2 blank,
+    // line 3 bad JSON, line 4 invalid plan. Reported numbers must be 3 and 4.
+    writeFileSync(file, `${VALID_PLAN}\n\nnot json at all\n${INVALID_PLAN}\n`, 'utf8');
+    const out: string[] = [];
+    const rejection = await runLint(
+      { profile: 'default', output: 'json', debug: false, plans: file },
+      { stdout: line => out.push(line) },
+    ).catch((error: unknown) => error);
+    expect(rejection).toMatchObject({ exitCode: 5 });
+    const report = JSON.parse(out.join('')) as { checked: number; issues: Array<{ file: string }> };
+    expect(report.checked).toBe(3);
+    expect(report.issues.some(issue => issue.file.endsWith(':3'))).toBe(true);
+    expect(report.issues.some(issue => issue.file.endsWith(':4'))).toBe(true);
+    // The blank line itself is not an entry and never reports.
+    expect(report.issues.some(issue => issue.file.endsWith(':2'))).toBe(false);
+  });
+
+  it('requires exactly one input source', async () => {
+    await expect(
+      runLint({ profile: 'default', output: 'json', debug: false }, { stdout: () => undefined }),
+    ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 });
+  });
+});
+
 describe('runResult', () => {
   it('JSON mode prints the §6.5 LatestResult shape verbatim', async () => {
     const { credentialsPath } = makeCreds();

src/commands/test.ts

@@ -3862,6 +3862,144 @@ function renderRunDiffText(diff: CliRunDiff): string {
   return lines.join('\n');
 }
 
+export interface LintOptions extends CommonOptions {
+  planFrom?: string;
+  planFromDir?: string;
+  plans?: string;
+  steps?: string;
+}
+
+export interface CliLintIssue {
+  file: string;
+  field: string;
+  reason: string;
+}
+
+export interface CliLintReport {
+  checked: number;
+  valid: number;
+  issues: CliLintIssue[];
+}
+
+/**
+ * `test lint` (issue #98): validate plan/steps files fully OFFLINE with the
+ * SAME validators the create paths run, but collecting EVERY problem instead
+ * of dying on the first one, and without any network write. The create-batch
+ * reader is first-error-fatal and only reachable through a command that POSTs,
+ * so authoring a 12-plan directory meant one error per paid round-trip. Zero
+ * network, zero credentials: exit 0 when everything is valid, 5 otherwise, so
+ * it drops into a pre-commit hook or CI step before `create-batch`.
+ */
+export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise<CliLintReport> {
+  const out = makeOutput(opts.output, deps);
+  const sources = [opts.planFrom, opts.planFromDir, opts.plans, opts.steps].filter(
+    source => source !== undefined,
+  );
+  if (sources.length !== 1) {
+    throw localValidationError(
+      'plan-from',
+      'exactly one of --plan-from, --plan-from-dir, --plans, or --steps is required',
+    );
+  }
+
+  const issues: CliLintIssue[] = [];
+  let checked = 0;
+  // Run one existing validator, converting its typed throw into a report row
+  // (same envelopes, so `details.field` pointers like planSteps[2].type
+  // survive verbatim).
+  const collect = (file: string, validate: () => void): void => {
+    checked += 1;
+    try {
+      validate();
+    } catch (err) {
+      if (err instanceof ApiError) {
+        issues.push({
+          file,
+          field: String(err.getDetail('field') ?? '(file)'),
+          reason: String(err.getDetail('reason') ?? err.nextAction ?? err.message),
+        });
+      } else {
+        issues.push({
+          file,
+          field: '(file)',
+          reason: err instanceof Error ? err.message : String(err),
+        });
+      }
+    }
+  };
+
+  if (opts.planFrom !== undefined) {
+    const planFrom = opts.planFrom;
+    collect(planFrom, () => void readPlanFromGuarded(planFrom));
+  } else if (opts.steps !== undefined) {
+    const steps = opts.steps;
+    collect(steps, () => void readPlanStepsFileGuarded(steps));
+  } else if (opts.planFromDir !== undefined) {
+    const dir = resolveAbsolute(opts.planFromDir);
+    let entries: string[];
+    try {
+      entries = readdirSync(dir)
+        .filter(name => name.endsWith('.json'))
+        .sort();
+    } catch {
+      throw localValidationError('plan-from-dir', `cannot read directory: ${dir}`);
+    }
+    if (entries.length === 0) {
+      throw localValidationError('plan-from-dir', 'contains no *.json plan files');
+    }
+    for (const entry of entries) {
+      collect(entry, () => void readPlanFromGuarded(join(dir, entry)));
+    }
+  } else if (opts.plans !== undefined) {
+    // JSONL: validate PER LINE so every bad line reports (the create path's
+    // reader stays throw-on-first; this is the collecting counterpart).
+    const absolute = resolveAbsolute(opts.plans);
+    let content: string;
+    try {
+      content = readFileSync(absolute, 'utf8');
+    } catch {
+      throw localValidationError('plans', `cannot read file: ${absolute}`);
+    }
+    // Index lines BEFORE dropping blanks so every reported `file:N` points at
+    // the PHYSICAL line in the file (a blank separator line must not shift all
+    // subsequent line numbers).
+    const numberedLines = content
+      .split('\n')
+      .map((rawLine, physicalIndex) => ({ line: rawLine.trim(), lineNo: physicalIndex + 1 }))
+      .filter(entry => entry.line.length > 0);
+    if (numberedLines.length === 0) throw localValidationError('plans', 'contains no plan lines');
+    for (const { line, lineNo } of numberedLines) {
+      collect(`${opts.plans}:${lineNo}`, () => {
+        let parsed: unknown;
+        try {
+          parsed = JSON.parse(line);
+        } catch {
+          throw localValidationError(
+            'plans',
+            `line ${lineNo} is not valid JSON`,
+            undefined,
+            'field',
+          );
+        }
+        assertPlanShape(parsed, { specIndex: lineNo - 1 });
+      });
+    }
+  }
+
+  const filesWithIssues = new Set(issues.map(issue => issue.file)).size;
+  const report: CliLintReport = { checked, valid: checked - filesWithIssues, issues };
+  out.print(report, () =>
+    [
+      ...issues.map(issue => `${issue.file}: ${issue.field}: ${issue.reason}`),
+      `${report.valid}/${report.checked} valid, ${issues.length} problem(s)`,
+    ].join('\n'),
+  );
+  if (issues.length > 0) {
+    throw new CLIError(`lint: ${issues.length} problem(s) across ${report.checked} file(s)`, 5);
+  }
+  return report;
+}
+
 export async function runSteps(
   opts: StepsOptions,
   deps: TestDeps = {},
@@ -7669,6 +7807,37 @@ export function createTestCommand(deps: TestDeps = {}): Command {
       await runDiff({ ...resolveCommonOptions(command), runA, runB }, deps);
     });
 
+  test
+    .command('lint')
+    .description(
+      'Validate plan/steps files offline with the same validators `create` runs, collecting EVERY problem. No network, no credentials. Exit 0 when all valid, 5 otherwise.',
+    )
+    .option('--plan-from <file>', 'single plan JSON file')
+    .option(
+      '--plan-from-dir <dir>',
+      'directory of *.json plan files (each checked, all errors reported)',
+    )
+    .option('--plans <file>', 'JSONL file with one plan spec per line (each line checked)')
+    .option('--steps <file>', 'plan-steps JSON file (the shape `test plan put` ingests)')
+    .addHelpText('after', GLOBAL_OPTS_HINT)
+    .action(
+      async (
+        cmdOpts: { planFrom?: string; planFromDir?: string; plans?: string; steps?: string },
+        command: Command,
+      ) => {
+        await runLint(
+          {
+            ...resolveCommonOptions(command),
+            planFrom: cmdOpts.planFrom,
+            planFromDir: cmdOpts.planFromDir,
+            plans: cmdOpts.plans,
+            steps: cmdOpts.steps,
+          },
+          deps,
+        );
+      },
+    );
+
   test
     .command('result <test-id>')
     .description(

test/__snapshots__/help.snapshot.test.ts.snap

@@ -208,6 +208,11 @@ Commands:
                                         failedStepIndex, per-step status flips,
                                         codeVersion drift. Exit 0 when verdicts
                                         match, 1 when they differ.
+  lint [options]                        Validate plan/steps files offline with
+                                        the same validators \`create\` runs,
+                                        collecting EVERY problem. No network,
+                                        no credentials. Exit 0 when all valid,
+                                        5 otherwise.
   result [options] <test-id>            Get the latest result for a test (default) or list prior runs (--history).
   
   --output json shape differs by mode: