feat(test): add "test scaffold" to emit a schema-correct starter plan or backend skeleton (#180)

Andy00L · web-flow · commit 3305dfa57ab2 · 2026-07-06T13:14:29.000-07:00
diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts
@@ -35,6 +35,7 @@ import {
   runList,
   runPlanPut,
   runResult,
+  runScaffold,
   runSteps,
   runTestWaitMany,
   runUpdate,
@@ -134,6 +135,7 @@ describe('createTestCommand — surface', () => {
       'rerun',
       'result',
       'run',
+      'scaffold',
       'steps',
       'update',
       'wait',
@@ -2325,6 +2327,94 @@ describe('runCodePut', () => {
   });
 });
 
+describe('runScaffold', () => {
+  it('frontend scaffold is a valid CliPlanInput and prints as JSON on stdout', async () => {
+    const out: string[] = [];
+    const result = await runScaffold(
+      { profile: 'default', output: 'text', debug: false, scaffoldType: 'frontend', force: false },
+      { stdout: line => out.push(line), stderr: () => undefined, env: {} },
+    );
+    const plan = result as { projectId: string; type: string; planSteps: Array<{ type: string }> };
+    expect(plan.type).toBe('frontend');
+    // Placeholder project id when TESTSPRITE_PROJECT_ID is unset.
+    expect(plan.projectId).toContain('testsprite project list');
+    expect(plan.planSteps.length).toBeGreaterThanOrEqual(2);
+    // Every emitted step type must come from the real enum (no drift).
+    for (const step of plan.planSteps) expect(['action', 'assertion']).toContain(step.type);
+    // At least one assertion step so the scaffold is a meaningful test.
+    expect(plan.planSteps.some(step => step.type === 'assertion')).toBe(true);
+    // stdout body parses back to the same plan (`> plan.json` works).
+    expect(JSON.parse(out.join('\n'))).toEqual(plan);
+  });
+
+  it('pre-fills projectId from TESTSPRITE_PROJECT_ID when set', async () => {
+    const result = await runScaffold(
+      { profile: 'default', output: 'json', debug: false, scaffoldType: 'frontend', force: false },
+      {
+        stdout: () => undefined,
+        stderr: () => undefined,
+        env: { TESTSPRITE_PROJECT_ID: 'project_env' },
+      },
+    );
+    expect((result as { projectId: string }).projectId).toBe('project_env');
+  });
+
+  it('backend scaffold defines a requests test with a status assertion AND calls it', async () => {
+    const out: string[] = [];
+    const result = await runScaffold(
+      { profile: 'default', output: 'text', debug: false, scaffoldType: 'backend', force: false },
+      { stdout: line => out.push(line), stderr: () => undefined, env: {} },
+    );
+    const code = (result as { code: string }).code;
+    expect(code).toContain('import requests');
+    expect(code).toContain('assert response.status_code == 200');
+    // The onboarding rule: the function must be CALLED, not just defined.
+    expect(code).toContain('\ntest_health_endpoint()');
+    expect(out.join('\n')).toContain('import requests');
+  });
+
+  it('--out writes the file and refuses to overwrite without --force', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-'));
+    const target = join(dir, 'plan.json');
+    const opts = {
+      profile: 'default',
+      output: 'text',
+      debug: false,
+      scaffoldType: 'frontend',
+      out: target,
+      force: false,
+    } as const;
+    const deps = { stdout: () => undefined, stderr: () => undefined, env: {} };
+    await runScaffold({ ...opts }, deps);
+    const written = JSON.parse(readFileSync(target, 'utf8')) as { type: string };
+    expect(written.type).toBe('frontend');
+    // Second run without --force must not clobber the (possibly edited) file.
+    await expect(runScaffold({ ...opts }, deps)).rejects.toMatchObject({
+      code: 'VALIDATION_ERROR',
+      exitCode: 5,
+    });
+    // --force overwrites.
+    await expect(runScaffold({ ...opts, force: true }, deps)).resolves.toBeDefined();
+  });
+
+  it('--out pointing at an existing path (here a directory) rejects without --force', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-dir-'));
+    await expect(
+      runScaffold(
+        {
+          profile: 'default',
+          output: 'text',
+          debug: false,
+          scaffoldType: 'frontend',
+          out: dir,
+          force: false,
+        },
+        { stdout: () => undefined, stderr: () => undefined, env: {} },
+      ),
+    ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 });
+  });
+});
+
 describe('runSteps', () => {
   it('JSON mode returns the §6.4 wire shape and forwards pageSize/cursor', async () => {
     const { credentialsPath } = makeCreds();
diff --git a/src/commands/test.ts b/src/commands/test.ts
@@ -1,4 +1,11 @@
-import { createWriteStream, readFileSync, readdirSync, statSync, type WriteStream } from 'node:fs';
+import {
+  createWriteStream,
+  existsSync,
+  readFileSync,
+  readdirSync,
+  statSync,
+  type WriteStream,
+} from 'node:fs';
 import { rename, stat, unlink } from 'node:fs/promises';
 import { basename, dirname, extname, isAbsolute, join, resolve } from 'node:path';
 import { randomUUID } from 'node:crypto';
@@ -4000,6 +4007,114 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise<C
   return report;
 }
 
+/** Flag options for `test scaffold`. */
+interface ScaffoldFlagOpts {
+  type?: string;
+  out?: string;
+  force?: boolean;
+}
+
+export interface ScaffoldOptions extends CommonOptions {
+  scaffoldType: 'frontend' | 'backend';
+  out?: string;
+  force: boolean;
+}
+
+/** JSON payload `test scaffold --type backend` prints under --output json. */
+export interface CliBackendScaffold {
+  type: 'backend';
+  language: 'python';
+  code: string;
+}
+
+/**
+ * `test scaffold` — emit a schema-correct starter test definition so a first
+ * test never starts from hand-copied JSON. Pure-local: no network, no
+ * credentials, no filesystem reads. The frontend template is a `CliPlanInput`
+ * (the exact shape `--plan-from` ingests; sourceRef: CliPlanInput /
+ * PLAN_STEP_TYPES above), so `scaffold | create --plan-from -`-style flows
+ * validate out of the box. The backend template is the minimal `requests`
+ * script the onboarding skill mandates: define a test function with a
+ * concrete status assertion, then CALL it (a defined-but-never-called test
+ * would pass without asserting anything).
+ */
+export async function runScaffold(
+  opts: ScaffoldOptions,
+  deps: TestDeps = {},
+): Promise<CliPlanInput | CliBackendScaffold> {
+  const out = makeOutput(opts.output, deps);
+  const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`));
+  const env = deps.env ?? process.env;
+  // Pre-fill the project id from TESTSPRITE_PROJECT_ID when the caller's
+  // environment carries one; otherwise a clearly-marked placeholder the user
+  // swaps after running `testsprite project list`.
+  const projectId =
+    typeof env.TESTSPRITE_PROJECT_ID === 'string' && env.TESTSPRITE_PROJECT_ID.length > 0
+      ? env.TESTSPRITE_PROJECT_ID
+      : '<run: testsprite project list>';
+
+  let payload: CliPlanInput | CliBackendScaffold;
+  let body: string;
+  if (opts.scaffoldType === 'frontend') {
+    const plan: CliPlanInput = {
+      projectId,
+      type: 'frontend',
+      name: 'My first frontend test',
+      description: 'Replace with one sentence describing what this test verifies.',
+      priority: 'p2',
+      planSteps: [
+        {
+          type: 'action',
+          description: 'Navigate to /login and sign in with a seeded test account',
+        },
+        { type: 'action', description: 'Open the first product page and click "Add to cart"' },
+        { type: 'assertion', description: 'Assert that the cart badge shows 1 item' },
+      ],
+    };
+    payload = plan;
+    body = `${JSON.stringify(plan, null, 2)}\n`;
+  } else {
+    const code = [
+      'import requests',
+      '',
+      '# Replace with your API base URL (must be reachable from the internet).',
+      'BASE_URL = "https://staging.example.com"',
+      '',
+      '',
+      'def test_health_endpoint() -> None:',
+      '    response = requests.get(f"{BASE_URL}/health", timeout=30)',
+      '    assert response.status_code == 200, f"expected 200, got {response.status_code}"',
+      '',
+      '',
+      '# The test function MUST be called: TestSprite executes this file top to',
+      '# bottom, so a defined-but-never-called function would pass vacuously.',
+      'test_health_endpoint()',
+      '',
+    ].join('\n');
+    payload = { type: 'backend', language: 'python', code };
+    body = code;
+  }
+
+  if (opts.out !== undefined) {
+    const resolved = isAbsolute(opts.out) ? opts.out : resolve(process.cwd(), opts.out);
+    // Never clobber silently: scaffolds are starting points the user edits, so
+    // an accidental re-run must not erase their work. --force opts in.
+    if (!opts.force && existsSync(resolved)) {
+      throw localValidationError('out', `already exists: ${resolved}. Pass --force to overwrite`);
+    }
+    const sink = openOutputFile(opts.out); // reuses the directory/parent guards
+    const fileOut = makeFileOutput(opts.output, sink);
+    await fileOut.writeChunk(body);
+    await closeOutputFile(sink, true);
+    stderrFn(`Scaffold written to ${resolved}`);
+    return payload;
+  }
+
+  // No --out: the scaffold body IS the stdout payload (`> plan.json` works).
+  out.print(payload, () => body.trimEnd());
+  return payload;
+}
+
 export async function runSteps(
   opts: StepsOptions,
   deps: TestDeps = {},
@@ -8007,6 +8122,34 @@ export function createTestCommand(deps: TestDeps = {}): Command {
       );
     });
 
+  test
+    .command('scaffold')
+    .description(
+      'Emit a schema-correct starter test definition (frontend plan JSON by default, or a backend Python skeleton). Pure-local: no network, no credentials.',
+    )
+    .option('--type <type>', 'frontend|backend (default: frontend)')
+    .option('--out <path>', 'write the scaffold to a file instead of stdout')
+    .option('--force', 'overwrite an existing --out file', false)
+    .addHelpText(
+      'after',
+      '\nExamples:\n' +
+        '  testsprite test scaffold > first-test.plan.json\n' +
+        '  testsprite test scaffold --type backend --out tests/health.py\n' +
+        '  testsprite test scaffold --out plan.json   # then edit, and create with --plan-from plan.json',
+    )
+    .addHelpText('after', GLOBAL_OPTS_HINT)
+    .action(async (cmdOpts: ScaffoldFlagOpts, command: Command) => {
+      await runScaffold(
+        {
+          ...resolveCommonOptions(command),
+          scaffoldType: parseEnumFlag(cmdOpts.type, 'type', TEST_TYPES) ?? 'frontend',
+          out: cmdOpts.out,
+          force: cmdOpts.force === true,
+        },
+        deps,
+      );
+    });
+
   test
     .command('steps <test-id>')
     .description(
diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap
@@ -201,6 +201,10 @@ Commands:
                                         (--plan-from, FE-only, M3.2 piece-5)
   create-batch [options]                Create multiple FE tests from a JSONL
                                         of plan specs (FE-only)
+  scaffold [options]                    Emit a schema-correct starter test
+                                        definition (frontend plan JSON by
+                                        default, or a backend Python skeleton).
+                                        Pure-local: no network, no credentials.
   steps [options] <test-id>             List the steps for a test (server
                                         returns the cumulative log across every
                                         run; use --run-id to scope to one run)
