fix(flaky): cap --runs at 10 per maintainer scope (#115)

Rescope the flaky detector's --runs bound from 1-100 to 1-10 as requested in the #115 triage: uncapped FE replays amplify free executions. Updates the MAX_FLAKY_RUNS constant (which drives the validation, error message, and --runs help text), docs, changelog, and the runs-bound tests. Regenerates the help snapshot, which also adds the previously-missing 'test flaky' entry.

Author: lxcario

CHANGELOG.md

@@ -6,7 +6,7 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for
 
 ### Added
 
-- **`testsprite test flaky <test-id>`** — repeat-run flaky-test detector. Replays a test N times (`--runs <n>`, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal OFF (strict verbatim) so a healed drift can't mask a nondeterministic pass/fail. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness (`testsprite test flaky <id> --runs 5 || exit 1`). Flags: `--runs <n>` (1–100), `--until-fail` (stop at the first non-passing attempt), `--timeout <s>` (per-attempt), and `--output json` for a machine-readable stability report. Frontend replays are free verbatim script replays; a one-line advisory is printed for backend tests, whose closure reruns may cost credits.
+- **`testsprite test flaky <test-id>`** — repeat-run flaky-test detector. Replays a test N times (`--runs <n>`, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal OFF (strict verbatim) so a healed drift can't mask a nondeterministic pass/fail. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness (`testsprite test flaky <id> --runs 5 || exit 1`). Flags: `--runs <n>` (1–10), `--until-fail` (stop at the first non-passing attempt), `--timeout <s>` (per-attempt), and `--output json` for a machine-readable stability report. Frontend replays are free verbatim script replays; a one-line advisory is printed for backend tests, whose closure reruns may cost credits.
 
 ## [0.1.2] - 2026-06-19
 

DOCUMENTATION.md

@@ -389,15 +389,15 @@ Detect a **flaky** test by replaying it several times and reporting how often it
 testsprite test flaky test_xxxxxxxx --runs 10
 
 # Fast "is it flaky at all?" — stop at the first non-passing attempt
-testsprite test flaky test_xxxxxxxx --runs 20 --until-fail
+testsprite test flaky test_xxxxxxxx --runs 10 --until-fail
 
 # Machine-readable stability report for CI
 testsprite test flaky test_xxxxxxxx --runs 10 --output json
 ```
 
 Flags:
 
-- `--runs <n>` — number of replays (1–100, default 5).
+- `--runs <n>` — number of replays (1–10, default 5).
 - `--until-fail` — stop at the first attempt that does not pass.
 - `--timeout <s>` — per-attempt polling deadline (same semantics as `test wait`).
 

src/commands/test.flaky.spec.ts

@@ -345,7 +345,7 @@ describe('runFlaky', () => {
     expect((err as ApiError).code).toBe('NOT_FOUND');
   });
 
-  it('rejects --runs outside 1..100 with a validation error (exit 5)', async () => {
+  it('rejects --runs below the range (0) with a validation error (exit 5)', async () => {
     const { fetchImpl } = makeFlakyFetch({ statuses: [] });
     const { deps } = makeDeps(fetchImpl);
     const err = await runFlaky(
@@ -365,4 +365,25 @@ describe('runFlaky', () => {
     expect(err).toBeInstanceOf(ApiError);
     expect((err as ApiError).exitCode).toBe(5);
   });
+
+  it('rejects --runs above the cap (11) with a validation error (exit 5)', async () => {
+    const { fetchImpl } = makeFlakyFetch({ statuses: [] });
+    const { deps } = makeDeps(fetchImpl);
+    const err = await runFlaky(
+      {
+        profile: 'default',
+        output: 'text',
+        dryRun: false,
+        debug: false,
+        verbose: false,
+        testId: 'test_x',
+        runs: 11,
+        untilFail: false,
+        timeoutSeconds: 600,
+      },
+      deps,
+    ).catch((e: unknown) => e);
+    expect(err).toBeInstanceOf(ApiError);
+    expect((err as ApiError).exitCode).toBe(5);
+  });
 });

src/commands/test.ts

@@ -7567,8 +7567,8 @@ export function createTestCommand(deps: TestDeps = {}): Command {
 // `test flaky` — repeat-run flaky-test detector
 // ---------------------------------------------------------------------------
 
-/** Upper bound on `--runs` so a typo can't spawn thousands of replays. */
-const MAX_FLAKY_RUNS = 100;
+/** Upper bound on `--runs` so a repeat-runner can't amplify free FE replays. */
+const MAX_FLAKY_RUNS = 10;
 /** Default replay count when `--runs` is omitted. */
 const DEFAULT_FLAKY_RUNS = 5;
 

test/__snapshots__/help.snapshot.test.ts.snap

@@ -259,6 +259,15 @@ Commands:
    11  rate limited — honor Retry-After
   
   On failure/blocked/cancelled, run: testsprite test artifact get <run-id>
+  flaky [options] <test-id>             Repeatedly replay a test to measure stability and surface flakiness.
+  Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail.
+  
+  Exit codes:
+    0  stable (every attempt passed)
+    1  flaky or failing (at least one attempt did not pass)
+    3  auth error
+    4  test not found (no replayable run — trigger \`testsprite test run <id>\` first)
+    5  validation error
   code                                  Inspect and edit generated test code
   plan                                  Manage FE test plan-steps (FE-only)
   failure                               Export the latest-failure agent bundle
@@ -334,6 +343,38 @@ Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --deb
 "
 `;
 
+exports[`--help snapshots > test flaky 1`] = `
+"Usage: testsprite test flaky [options] <test-id>
+
+Repeatedly replay a test to measure stability and surface flakiness.
+Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail.
+
+Exit codes:
+  0  stable (every attempt passed)
+  1  flaky or failing (at least one attempt did not pass)
+  3  auth error
+  4  test not found (no replayable run — trigger \`testsprite test run <id>\` first)
+  5  validation error
+
+Options:
+  --runs <n>     number of replays to run (1-10, default 5)
+  --until-fail   stop at the first non-passing attempt (fast "is it flaky at
+                 all?" check) (default: false)
+  --timeout <s>  per-attempt max seconds to wait (1-3600, default 600)
+  -h, --help     display help for command
+
+Notes:
+  • Frontend replays are free verbatim script replays (no credit); backend replays
+    re-run the dependency closure and may cost credits — a one-line advisory is printed.
+  • Replays use auto-heal OFF so a flaky test is not silently "healed" into a pass;
+    this measures replay stability of the saved script against the configured URL.
+  • \`--output json\` emits a machine-readable stability report for CI gating.
+
+Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug):
+  testsprite --help
+"
+`;
+
 exports[`--help snapshots > test get 1`] = `
 "Usage: testsprite test get [options] <test-id>