Skip to content

Commit 0bf2513

Browse files
committed
feat(@probitas/probitas): add --failed flag to re-run failed scenarios
Adds -F/--failed option to `probitas run` command that filters execution to only scenarios that failed in the previous run. State is persisted to .probitas/last-run.json after each run. - Add state persistence module (src/cli/state.ts) with load/save functions - Add FailedScenarioFilter to run protocol for subprocess communication - Apply failed filter after selector filtering (AND logic) - Add .probitas/ to .gitignore for machine-specific state
1 parent 9ee2fbe commit 0bf2513

8 files changed

Lines changed: 542 additions & 2 deletions

File tree

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
.DS_Store
22
.coverage
3+
.probitas/
34
result
45
dist/
56
src/cli/_embedded_templates.ts

README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,9 @@ probitas run
200200
# Run scenarios with specific tag
201201
probitas run -s tag:api
202202

203+
# Re-run only failed scenarios from previous run
204+
probitas run --failed
205+
203206
# Run with JSON output
204207
probitas run --reporter json
205208

assets/usage-run.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ Options:
2121
-S, --sequential Run scenarios sequentially (alias for --max-concurrency=1)
2222
--max-failures <n> Stop after N failures
2323
-f, --fail-fast Stop on first failure (alias for --max-failures=1)
24+
-F, --failed Run only scenarios that failed in the previous run
25+
Reads from .probitas/last-run.json
2426
--config <path> Path to config file (deno.json/deno.jsonc)
2527
--env <file> Load environment variables from file (default: .env)
2628
--no-env Skip loading .env file
@@ -93,6 +95,11 @@ Examples:
9395
probitas run -f Stop on first failure (alias)
9496
probitas run --reload Reload dependencies before running
9597

98+
# Re-run failed scenarios
99+
probitas run --failed Run only scenarios that failed previously
100+
probitas run -F Run only scenarios that failed previously (short)
101+
probitas run -F -s tag:api Run failed scenarios AND with @api tag
102+
96103
# Output control
97104
probitas run -v Verbose output
98105
probitas run --verbose Verbose output (same as -v)

src/cli/_templates/run.ts

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ runSubprocess<RunCommandInput>(async (ipc, input) => {
6666
timeout,
6767
stepOptions,
6868
logLevel,
69+
failedFilter,
6970
} = input;
7071

7172
// Create abort controller for this run
@@ -75,8 +76,8 @@ runSubprocess<RunCommandInput>(async (ipc, input) => {
7576
await configureLogging(logLevel);
7677

7778
try {
78-
// Load scenarios from files
79-
const scenarios = applySelectors(
79+
// Load scenarios from files and apply selectors
80+
let scenarios = applySelectors(
8081
await loadScenarios(filePaths, {
8182
onImportError: (file, err) => {
8283
const m = err instanceof Error ? err.message : String(err);
@@ -86,6 +87,26 @@ runSubprocess<RunCommandInput>(async (ipc, input) => {
8687
selectors,
8788
);
8889

90+
// Apply failed filter if provided (for --failed flag)
91+
if (failedFilter && failedFilter.length > 0) {
92+
const failedSet = new Set(
93+
failedFilter.map((f) => `${f.name}|${f.file}`),
94+
);
95+
scenarios = scenarios.filter((s) => {
96+
const scenarioFile = s.origin?.path ?? "unknown";
97+
// Check both absolute path and relative path matching
98+
return failedSet.has(`${s.name}|${scenarioFile}`) ||
99+
failedFilter.some((f) =>
100+
f.name === s.name && scenarioFile.endsWith(f.file)
101+
);
102+
});
103+
104+
logger.debug("Applied failed filter", {
105+
filterCount: failedFilter.length,
106+
matchedCount: scenarios.length,
107+
});
108+
}
109+
89110
// Check if aborted during loading phase
90111
if (globalAbortController.signal.aborted) {
91112
throw new Error("Aborted during scenario loading");
@@ -96,6 +117,7 @@ runSubprocess<RunCommandInput>(async (ipc, input) => {
96117
logger.info("No scenarios found after applying selectors", {
97118
filePaths,
98119
selectors,
120+
failedFilter: failedFilter?.length ?? 0,
99121
});
100122
}
101123

src/cli/_templates/run_protocol.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,16 @@ export type RunInput =
3434
| RunCommandInput
3535
| RunAbortInput;
3636

37+
/**
38+
* Failed scenario identifier for filtering
39+
*/
40+
export interface FailedScenarioFilter {
41+
/** Scenario name */
42+
readonly name: string;
43+
/** File path (relative or absolute) */
44+
readonly file: string;
45+
}
46+
3747
/**
3848
* Run scenarios command
3949
*/
@@ -53,6 +63,8 @@ export interface RunCommandInput {
5363
readonly stepOptions?: StepOptions;
5464
/** Log level for subprocess logging */
5565
readonly logLevel: LogLevel;
66+
/** Filter to only run these failed scenarios (name + file pairs) */
67+
readonly failedFilter?: readonly FailedScenarioFilter[];
5668
}
5769

5870
/**

src/cli/commands/run.ts

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,12 @@ import {
3434
deserializeRunResult,
3535
deserializeScenarioResult,
3636
deserializeStepResult,
37+
type FailedScenarioFilter,
3738
isRunOutput,
3839
type RunCommandInput,
3940
type RunOutput,
4041
} from "../_templates/run_protocol.ts";
42+
import { loadLastRunState, saveLastRunState } from "../state.ts";
4143

4244
const logger = getLogger(["probitas", "cli", "run"]);
4345

@@ -67,13 +69,15 @@ const PARSE_ARGS_CONFIG = {
6769
"debug",
6870
"sequential",
6971
"fail-fast",
72+
"failed",
7073
],
7174
collect: ["include", "exclude", "selector"],
7275
alias: {
7376
h: "help",
7477
s: "selector",
7578
S: "sequential",
7679
f: "fail-fast",
80+
F: "failed",
7781
v: "verbose",
7882
q: "quiet",
7983
d: "debug",
@@ -220,6 +224,29 @@ export async function runCommand(
220224
// Get selectors (will be applied in subprocess)
221225
const selectors = parsed.selector ?? config?.selectors ?? [];
222226

227+
// Handle --failed flag: load previous run state and build filter
228+
let failedFilter: FailedScenarioFilter[] | undefined;
229+
if (parsed.failed) {
230+
const lastRunState = await loadLastRunState(cwd);
231+
if (!lastRunState) {
232+
console.warn(
233+
"No previous run state found. Running all matching scenarios.",
234+
);
235+
} else if (lastRunState.failed.length === 0) {
236+
console.log("No failed scenarios from previous run.");
237+
return EXIT_CODE.SUCCESS;
238+
} else {
239+
failedFilter = lastRunState.failed.map((f) => ({
240+
name: f.name,
241+
file: f.file,
242+
}));
243+
logger.debug("Loaded failed filter from previous run", {
244+
count: failedFilter.length,
245+
scenarios: failedFilter,
246+
});
247+
}
248+
}
249+
223250
// Parse options
224251
const maxConcurrency = parsed.sequential
225252
? 1
@@ -250,6 +277,7 @@ export async function runCommand(
250277
const runResult = await runWithSubprocess(scenarioFiles, {
251278
reporter,
252279
selectors,
280+
failedFilter,
253281
maxConcurrency: maxConcurrency ?? 0,
254282
maxFailures: maxFailures ?? 0,
255283
timeout,
@@ -267,6 +295,9 @@ export async function runCommand(
267295
skipped: runResult.skipped,
268296
});
269297

298+
// Save run state for --failed flag support
299+
await saveLastRunState(cwd, runResult);
300+
270301
return runResult.failed > 0 ? EXIT_CODE.FAILURE : EXIT_CODE.SUCCESS;
271302
} catch (err: unknown) {
272303
logger.error("Unexpected error in run command", { error: err });
@@ -286,6 +317,7 @@ async function runWithSubprocess(
286317
options: {
287318
reporter: Reporter;
288319
selectors: readonly string[];
320+
failedFilter?: readonly FailedScenarioFilter[];
289321
maxConcurrency: number;
290322
maxFailures: number;
291323
timeout: number;
@@ -299,6 +331,7 @@ async function runWithSubprocess(
299331
const {
300332
reporter,
301333
selectors,
334+
failedFilter,
302335
maxConcurrency,
303336
maxFailures,
304337
timeout,
@@ -324,6 +357,7 @@ async function runWithSubprocess(
324357
type: "run",
325358
filePaths,
326359
selectors,
360+
failedFilter,
327361
maxConcurrency,
328362
maxFailures,
329363
timeout,

src/cli/state.ts

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/**
2+
* State persistence for CLI commands
3+
*
4+
* Manages the `.probitas/` directory and stores run state between executions.
5+
*
6+
* @module
7+
*/
8+
9+
import { join } from "@std/path";
10+
import { relative } from "@std/path/relative";
11+
import type { RunResult } from "@probitas/runner";
12+
13+
/**
14+
* A scenario that failed in a previous run
15+
*/
16+
export interface FailedScenario {
17+
/** Scenario name */
18+
readonly name: string;
19+
/** Relative file path from project root */
20+
readonly file: string;
21+
/** Error message (optional, for display purposes) */
22+
readonly error?: string;
23+
}
24+
25+
/**
26+
* State of the last run
27+
*/
28+
export interface LastRunState {
29+
/** Schema version for forward compatibility */
30+
readonly version: 1;
31+
/** ISO timestamp of when the run completed */
32+
readonly timestamp: string;
33+
/** List of failed scenarios */
34+
readonly failed: readonly FailedScenario[];
35+
}
36+
37+
const STATE_DIR_NAME = ".probitas";
38+
const LAST_RUN_FILE = "last-run.json";
39+
const CURRENT_VERSION = 1;
40+
41+
/**
42+
* Get the state directory path, creating it if it doesn't exist
43+
*
44+
* @param cwd - Project root directory
45+
* @returns Path to the state directory
46+
*/
47+
export async function getStateDir(cwd: string): Promise<string> {
48+
const stateDir = join(cwd, STATE_DIR_NAME);
49+
try {
50+
await Deno.mkdir(stateDir, { recursive: true });
51+
} catch (error) {
52+
// Ignore if already exists
53+
if (!(error instanceof Deno.errors.AlreadyExists)) {
54+
throw error;
55+
}
56+
}
57+
return stateDir;
58+
}
59+
60+
/**
61+
* Save the last run state to disk
62+
*
63+
* Extracts failed scenarios from the run result and persists them
64+
* to `.probitas/last-run.json`.
65+
*
66+
* @param cwd - Project root directory
67+
* @param result - Run result from scenario execution
68+
*/
69+
export async function saveLastRunState(
70+
cwd: string,
71+
result: RunResult,
72+
): Promise<void> {
73+
const stateDir = await getStateDir(cwd);
74+
const statePath = join(stateDir, LAST_RUN_FILE);
75+
76+
// Extract failed scenarios from result
77+
const failed: FailedScenario[] = [];
78+
for (const s of result.scenarios) {
79+
if (s.status !== "failed") continue;
80+
81+
// Now TypeScript knows s.status is "failed" and s.error exists
82+
const metadata = s.metadata;
83+
const filePath = metadata.origin?.path ?? "unknown";
84+
const relativeFile = filePath !== "unknown"
85+
? relative(cwd, filePath)
86+
: "unknown";
87+
88+
failed.push({
89+
name: metadata.name,
90+
file: relativeFile,
91+
error: s.error instanceof Error
92+
? s.error.message
93+
: typeof s.error === "string"
94+
? s.error
95+
: undefined,
96+
});
97+
}
98+
99+
const state: LastRunState = {
100+
version: CURRENT_VERSION,
101+
timestamp: new Date().toISOString(),
102+
failed,
103+
};
104+
105+
await Deno.writeTextFile(statePath, JSON.stringify(state, null, 2) + "\n");
106+
}
107+
108+
/**
109+
* Load the last run state from disk
110+
*
111+
* @param cwd - Project root directory
112+
* @returns The last run state, or undefined if no state file exists or it's invalid
113+
*/
114+
export async function loadLastRunState(
115+
cwd: string,
116+
): Promise<LastRunState | undefined> {
117+
const statePath = join(cwd, STATE_DIR_NAME, LAST_RUN_FILE);
118+
119+
try {
120+
const content = await Deno.readTextFile(statePath);
121+
const state = JSON.parse(content);
122+
123+
// Validate version
124+
if (state.version !== CURRENT_VERSION) {
125+
return undefined;
126+
}
127+
128+
// Basic validation of required fields
129+
if (
130+
typeof state.timestamp !== "string" ||
131+
!Array.isArray(state.failed)
132+
) {
133+
return undefined;
134+
}
135+
136+
return state as LastRunState;
137+
} catch (error) {
138+
// File doesn't exist or can't be read
139+
if (error instanceof Deno.errors.NotFound) {
140+
return undefined;
141+
}
142+
// Invalid JSON or other error - treat as missing state
143+
if (error instanceof SyntaxError) {
144+
return undefined;
145+
}
146+
throw error;
147+
}
148+
}

0 commit comments

Comments
 (0)