Skip to content

Commit 6c806d0

Browse files
Andy00LAndy00L
committed
feat(test): add "test scaffold" to emit a schema-correct starter plan or backend skeleton
1 parent e53257d commit 6c806d0

3 files changed

Lines changed: 238 additions & 1 deletion

File tree

‎src/commands/test.test.ts‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import {
3333
runList,
3434
runPlanPut,
3535
runResult,
36+
runScaffold,
3637
runSteps,
3738
runUpdate,
3839
} from './test.js';
@@ -128,6 +129,7 @@ describe('createTestCommand — surface', () => {
128129
'rerun',
129130
'result',
130131
'run',
132+
'scaffold',
131133
'steps',
132134
'update',
133135
'wait',
@@ -2266,6 +2268,94 @@ describe('runCodePut', () => {
22662268
});
22672269
});
22682270

2271+
describe('runScaffold', () => {
2272+
it('frontend scaffold is a valid CliPlanInput and prints as JSON on stdout', async () => {
2273+
const out: string[] = [];
2274+
const result = await runScaffold(
2275+
{ profile: 'default', output: 'text', debug: false, scaffoldType: 'frontend', force: false },
2276+
{ stdout: line => out.push(line), stderr: () => undefined, env: {} },
2277+
);
2278+
const plan = result as { projectId: string; type: string; planSteps: Array<{ type: string }> };
2279+
expect(plan.type).toBe('frontend');
2280+
// Placeholder project id when TESTSPRITE_PROJECT_ID is unset.
2281+
expect(plan.projectId).toContain('testsprite project list');
2282+
expect(plan.planSteps.length).toBeGreaterThanOrEqual(2);
2283+
// Every emitted step type must come from the real enum (no drift).
2284+
for (const step of plan.planSteps) expect(['action', 'assertion']).toContain(step.type);
2285+
// At least one assertion step so the scaffold is a meaningful test.
2286+
expect(plan.planSteps.some(step => step.type === 'assertion')).toBe(true);
2287+
// stdout body parses back to the same plan (`> plan.json` works).
2288+
expect(JSON.parse(out.join('\n'))).toEqual(plan);
2289+
});
2290+
2291+
it('pre-fills projectId from TESTSPRITE_PROJECT_ID when set', async () => {
2292+
const result = await runScaffold(
2293+
{ profile: 'default', output: 'json', debug: false, scaffoldType: 'frontend', force: false },
2294+
{
2295+
stdout: () => undefined,
2296+
stderr: () => undefined,
2297+
env: { TESTSPRITE_PROJECT_ID: 'project_env' },
2298+
},
2299+
);
2300+
expect((result as { projectId: string }).projectId).toBe('project_env');
2301+
});
2302+
2303+
it('backend scaffold defines a requests test with a status assertion AND calls it', async () => {
2304+
const out: string[] = [];
2305+
const result = await runScaffold(
2306+
{ profile: 'default', output: 'text', debug: false, scaffoldType: 'backend', force: false },
2307+
{ stdout: line => out.push(line), stderr: () => undefined, env: {} },
2308+
);
2309+
const code = (result as { code: string }).code;
2310+
expect(code).toContain('import requests');
2311+
expect(code).toContain('assert response.status_code == 200');
2312+
// The onboarding rule: the function must be CALLED, not just defined.
2313+
expect(code).toContain('\ntest_health_endpoint()');
2314+
expect(out.join('\n')).toContain('import requests');
2315+
});
2316+
2317+
it('--out writes the file and refuses to overwrite without --force', async () => {
2318+
const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-'));
2319+
const target = join(dir, 'plan.json');
2320+
const opts = {
2321+
profile: 'default',
2322+
output: 'text',
2323+
debug: false,
2324+
scaffoldType: 'frontend',
2325+
out: target,
2326+
force: false,
2327+
} as const;
2328+
const deps = { stdout: () => undefined, stderr: () => undefined, env: {} };
2329+
await runScaffold({ ...opts }, deps);
2330+
const written = JSON.parse(readFileSync(target, 'utf8')) as { type: string };
2331+
expect(written.type).toBe('frontend');
2332+
// Second run without --force must not clobber the (possibly edited) file.
2333+
await expect(runScaffold({ ...opts }, deps)).rejects.toMatchObject({
2334+
code: 'VALIDATION_ERROR',
2335+
exitCode: 5,
2336+
});
2337+
// --force overwrites.
2338+
await expect(runScaffold({ ...opts, force: true }, deps)).resolves.toBeDefined();
2339+
});
2340+
2341+
it('--out pointing at an existing path (here a directory) rejects without --force', async () => {
2342+
const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-dir-'));
2343+
await expect(
2344+
runScaffold(
2345+
{
2346+
profile: 'default',
2347+
output: 'text',
2348+
debug: false,
2349+
scaffoldType: 'frontend',
2350+
out: dir,
2351+
force: false,
2352+
},
2353+
{ stdout: () => undefined, stderr: () => undefined, env: {} },
2354+
),
2355+
).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 });
2356+
});
2357+
});
2358+
22692359
describe('runSteps', () => {
22702360
it('JSON mode returns the §6.4 wire shape and forwards pageSize/cursor', async () => {
22712361
const { credentialsPath } = makeCreds();

‎src/commands/test.ts‎

Lines changed: 144 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,11 @@
1-
import { createWriteStream, readFileSync, readdirSync, statSync, type WriteStream } from 'node:fs';
1+
import {
2+
createWriteStream,
3+
existsSync,
4+
readFileSync,
5+
readdirSync,
6+
statSync,
7+
type WriteStream,
8+
} from 'node:fs';
29
import { rename, stat, unlink } from 'node:fs/promises';
310
import { basename, dirname, extname, isAbsolute, join, resolve } from 'node:path';
411
import { randomUUID } from 'node:crypto';
@@ -3630,6 +3637,114 @@ function mapRunStepToCliTestStep(step: RunStepDto, run: RunResponse): CliTestSte
36303637
};
36313638
}
36323639

3640+
/** Flag options for `test scaffold`. */
3641+
interface ScaffoldFlagOpts {
3642+
type?: string;
3643+
out?: string;
3644+
force?: boolean;
3645+
}
3646+
3647+
export interface ScaffoldOptions extends CommonOptions {
3648+
scaffoldType: 'frontend' | 'backend';
3649+
out?: string;
3650+
force: boolean;
3651+
}
3652+
3653+
/** JSON payload `test scaffold --type backend` prints under --output json. */
3654+
export interface CliBackendScaffold {
3655+
type: 'backend';
3656+
language: 'python';
3657+
code: string;
3658+
}
3659+
3660+
/**
3661+
* `test scaffold` — emit a schema-correct starter test definition so a first
3662+
* test never starts from hand-copied JSON. Pure-local: no network, no
3663+
* credentials, no filesystem reads. The frontend template is a `CliPlanInput`
3664+
* (the exact shape `--plan-from` ingests; sourceRef: CliPlanInput /
3665+
* PLAN_STEP_TYPES above), so `scaffold | create --plan-from -`-style flows
3666+
* validate out of the box. The backend template is the minimal `requests`
3667+
* script the onboarding skill mandates: define a test function with a
3668+
* concrete status assertion, then CALL it (a defined-but-never-called test
3669+
* would pass without asserting anything).
3670+
*/
3671+
export async function runScaffold(
3672+
opts: ScaffoldOptions,
3673+
deps: TestDeps = {},
3674+
): Promise<CliPlanInput | CliBackendScaffold> {
3675+
const out = makeOutput(opts.output, deps);
3676+
const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`));
3677+
const env = deps.env ?? process.env;
3678+
// Pre-fill the project id from TESTSPRITE_PROJECT_ID when the caller's
3679+
// environment carries one; otherwise a clearly-marked placeholder the user
3680+
// swaps after running `testsprite project list`.
3681+
const projectId =
3682+
typeof env.TESTSPRITE_PROJECT_ID === 'string' && env.TESTSPRITE_PROJECT_ID.length > 0
3683+
? env.TESTSPRITE_PROJECT_ID
3684+
: '<run: testsprite project list>';
3685+
3686+
let payload: CliPlanInput | CliBackendScaffold;
3687+
let body: string;
3688+
if (opts.scaffoldType === 'frontend') {
3689+
const plan: CliPlanInput = {
3690+
projectId,
3691+
type: 'frontend',
3692+
name: 'My first frontend test',
3693+
description: 'Replace with one sentence describing what this test verifies.',
3694+
priority: 'p2',
3695+
planSteps: [
3696+
{
3697+
type: 'action',
3698+
description: 'Navigate to /login and sign in with a seeded test account',
3699+
},
3700+
{ type: 'action', description: 'Open the first product page and click "Add to cart"' },
3701+
{ type: 'assertion', description: 'Assert that the cart badge shows 1 item' },
3702+
],
3703+
};
3704+
payload = plan;
3705+
body = `${JSON.stringify(plan, null, 2)}\n`;
3706+
} else {
3707+
const code = [
3708+
'import requests',
3709+
'',
3710+
'# Replace with your API base URL (must be reachable from the internet).',
3711+
'BASE_URL = "https://staging.example.com"',
3712+
'',
3713+
'',
3714+
'def test_health_endpoint() -> None:',
3715+
' response = requests.get(f"{BASE_URL}/health", timeout=30)',
3716+
' assert response.status_code == 200, f"expected 200, got {response.status_code}"',
3717+
'',
3718+
'',
3719+
'# The test function MUST be called: TestSprite executes this file top to',
3720+
'# bottom, so a defined-but-never-called function would pass vacuously.',
3721+
'test_health_endpoint()',
3722+
'',
3723+
].join('\n');
3724+
payload = { type: 'backend', language: 'python', code };
3725+
body = code;
3726+
}
3727+
3728+
if (opts.out !== undefined) {
3729+
const resolved = isAbsolute(opts.out) ? opts.out : resolve(process.cwd(), opts.out);
3730+
// Never clobber silently: scaffolds are starting points the user edits, so
3731+
// an accidental re-run must not erase their work. --force opts in.
3732+
if (!opts.force && existsSync(resolved)) {
3733+
throw localValidationError('out', `already exists: ${resolved}. Pass --force to overwrite`);
3734+
}
3735+
const sink = openOutputFile(opts.out); // reuses the directory/parent guards
3736+
const fileOut = makeFileOutput(opts.output, sink);
3737+
await fileOut.writeChunk(body);
3738+
await closeOutputFile(sink, true);
3739+
stderrFn(`Scaffold written to ${resolved}`);
3740+
return payload;
3741+
}
3742+
3743+
// No --out: the scaffold body IS the stdout payload (`> plan.json` works).
3744+
out.print(payload, () => body.trimEnd());
3745+
return payload;
3746+
}
3747+
36333748
export async function runSteps(
36343749
opts: StepsOptions,
36353750
deps: TestDeps = {},
@@ -7207,6 +7322,34 @@ export function createTestCommand(deps: TestDeps = {}): Command {
72077322
);
72087323
});
72097324

7325+
test
7326+
.command('scaffold')
7327+
.description(
7328+
'Emit a schema-correct starter test definition (frontend plan JSON by default, or a backend Python skeleton). Pure-local: no network, no credentials.',
7329+
)
7330+
.option('--type <type>', 'frontend|backend (default: frontend)')
7331+
.option('--out <path>', 'write the scaffold to a file instead of stdout')
7332+
.option('--force', 'overwrite an existing --out file', false)
7333+
.addHelpText(
7334+
'after',
7335+
'\nExamples:\n' +
7336+
' testsprite test scaffold > first-test.plan.json\n' +
7337+
' testsprite test scaffold --type backend --out tests/health.py\n' +
7338+
' testsprite test scaffold --out plan.json # then edit, and create with --plan-from plan.json',
7339+
)
7340+
.addHelpText('after', GLOBAL_OPTS_HINT)
7341+
.action(async (cmdOpts: ScaffoldFlagOpts, command: Command) => {
7342+
await runScaffold(
7343+
{
7344+
...resolveCommonOptions(command),
7345+
scaffoldType: parseEnumFlag(cmdOpts.type, 'type', TEST_TYPES) ?? 'frontend',
7346+
out: cmdOpts.out,
7347+
force: cmdOpts.force === true,
7348+
},
7349+
deps,
7350+
);
7351+
});
7352+
72107353
test
72117354
.command('steps <test-id>')
72127355
.description(

‎test/__snapshots__/help.snapshot.test.ts.snap‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,10 @@ Commands:
196196
(--plan-from, FE-only, M3.2 piece-5)
197197
create-batch [options] Create multiple FE tests from a JSONL
198198
of plan specs (FE-only)
199+
scaffold [options] Emit a schema-correct starter test
200+
definition (frontend plan JSON by
201+
default, or a backend Python skeleton).
202+
Pure-local: no network, no credentials.
199203
steps [options] <test-id> List the steps for a test (server
200204
returns the cumulative log across every
201205
run; use --run-id to scope to one run)

0 commit comments

Comments
 (0)