Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 90 additions & 0 deletions .github/workflows/mutation.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
name: Mutation Testing

# Mutation testing is slow (it re-runs the test suite per mutant), so it runs
# nightly and on demand — never on pull requests — to avoid PR latency. It is
# informational/non-blocking: the score is reported to the run summary and the
# HTML report is uploaded as an artifact.

on:
schedule:
- cron: "0 4 * * *" # nightly, 04:00 UTC
workflow_dispatch:
inputs:
mutate:
description: "Optional glob to scope mutation (e.g. src/tokens/**/*.ts)"
required: false
type: string

concurrency:
group: mutation-${{ github.ref }}
cancel-in-progress: true

jobs:
stryker:
name: Stryker (packages/pipeline)
runs-on: ubuntu-latest
timeout-minutes: 90
steps:
- uses: actions/checkout@v4

- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: "20"

- name: Install pnpm
uses: pnpm/action-setup@v4
with:
version: 9

- name: Install dependencies
run: pnpm install --frozen-lockfile

- name: Run Stryker mutation testing
working-directory: packages/pipeline
run: |
# Run from packages/pipeline so the pnpm-linked node_modules resolve in
# the Stryker sandbox; the config lives at the repo root.
if [ -n "${{ github.event.inputs.mutate }}" ]; then
npx stryker run ../../stryker.conf.json --mutate "${{ github.event.inputs.mutate }}"
else
npx stryker run ../../stryker.conf.json
fi

- name: Mutation score summary
if: always()
working-directory: packages/pipeline
run: |
node -e '
const fs = require("fs");
const path = "reports/mutation/mutation-report.json";
if (!fs.existsSync(path)) { console.log("No mutation report produced."); process.exit(0); }
const report = JSON.parse(fs.readFileSync(path, "utf-8"));
const counts = { Killed: 0, Survived: 0, Timeout: 0, NoCoverage: 0, RuntimeError: 0, CompileError: 0, Ignored: 0 };
for (const file of Object.values(report.files || {})) {
for (const m of file.mutants || []) counts[m.status] = (counts[m.status] || 0) + 1;
}
const detected = counts.Killed + counts.Timeout;
const valid = detected + counts.Survived + counts.NoCoverage;
const score = valid ? ((detected / valid) * 100).toFixed(2) : "n/a";
const out = [
"## Mutation testing (packages/pipeline)",
"",
`**Mutation score: ${score}%**` + (score === "n/a" ? "" : ` (target 80%, opt-in)`),
"",
"| Status | Count |",
"| --- | --- |",
...Object.entries(counts).map(([k, v]) => `| ${k} | ${v} |`),
].join("\n");
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY || "/dev/stdout", out + "\n");
console.log(`Mutation score: ${score}%`);
'

- name: Upload mutation report
if: always()
uses: actions/upload-artifact@v4
with:
name: mutation-report
path: packages/pipeline/reports/mutation/
if-no-files-found: ignore
retention-days: 14
27 changes: 27 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,33 @@ Full catalog: `.claude/skills/README.md`
./scripts/setup-playwright.sh # One-time browser engine setup
```

### Mutation testing (Stryker)

Mutation testing measures how well the test suite actually catches bugs. The
framework's own TypeScript code (`packages/pipeline`) is mutation-tested via
[Stryker](https://stryker-mutator.io/); `stryker.conf.json` lives at the repo
root but **runs from `packages/pipeline`** so the pnpm-linked dependencies resolve
inside Stryker's sandbox.

```bash
# Full run (slow — re-runs the suite per mutant)
cd packages/pipeline && npx stryker run ../../stryker.conf.json

# Scope to one area while iterating
cd packages/pipeline && npx stryker run ../../stryker.conf.json --mutate "src/tokens/**/*.ts"
```

The HTML report is written to `packages/pipeline/reports/mutation/index.html`.

- **Threshold (opt-in).** The target mutation score lives in
`.claude/pipeline.config.json` → `qualityGate.mutationScore`
(`{ "enabled": false, "threshold": 80, "blocking": false }`). It is **off by
default**; set `enabled: true` to treat it as a quality gate.
- **CI.** `.github/workflows/mutation.yml` runs Stryker **nightly** and
**on demand** (`workflow_dispatch`) — never on PRs, to avoid latency. It is
informational: the score is posted to the run summary and the HTML report is
uploaded as an artifact.

### Pipeline Verification
```bash
./scripts/verify-tokens.sh # Catch hardcoded design values
Expand Down
17 changes: 17 additions & 0 deletions packages/pipeline/vitest.stryker.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import { configDefaults, defineConfig } from "vitest/config";

/**
* Vitest config used by Stryker mutation runs (see ../../stryker.conf.json).
*
* Identical to vitest.config.ts except it excludes the end-to-end test, which
* reads a fixture at the repo root — outside the Stryker sandbox (cwd =
* packages/pipeline) — and would fail the initial test run. Unit tests provide
* the mutant-killing signal.
*/
export default defineConfig({
test: {
include: ["src/**/*.test.ts"],
environment: "node",
exclude: [...configDefaults.exclude, "**/__tests__/e2e.test.ts"],
},
});
28 changes: 28 additions & 0 deletions stryker.conf.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"$schema": "https://raw.githubusercontent.com/stryker-mutator/stryker/master/packages/core/schema/stryker-schema.json",
"_comment": "Mutation testing for the framework's own TypeScript code in packages/pipeline. Run from packages/pipeline (see .github/workflows/mutation.yml and the README) so the pnpm-linked node_modules resolve inside the Stryker sandbox.",
"packageManager": "pnpm",
"plugins": ["@stryker-mutator/vitest-runner", "@stryker-mutator/typescript-checker"],
"testRunner": "vitest",
"vitest": { "configFile": "vitest.stryker.config.ts" },
"checkers": ["typescript"],
"tsconfigFile": "tsconfig.json",
"mutate": [
"src/**/*.ts",
"!src/**/*.test.ts",
"!src/**/__tests__/**",
"!src/**/*.d.ts",
"!src/**/index.ts",
"!src/**/cli.ts",
"!src/**/pipeline-cli.ts"
],
"reporters": ["clear-text", "json", "html"],
"htmlReporter": { "fileName": "reports/mutation/index.html" },
"jsonReporter": { "fileName": "reports/mutation/mutation-report.json" },
"thresholds": { "high": 80, "low": 60, "break": null },
"concurrency": 2,
"timeoutMS": 60000,
"tempDirName": ".stryker-tmp",
"cleanTempDir": true,
"ignoreStatic": true
}
Loading