diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml new file mode 100644 index 0000000..821ae29 --- /dev/null +++ b/.github/workflows/mutation.yml @@ -0,0 +1,90 @@ +name: Mutation Testing + +# Mutation testing is slow (it re-runs the test suite per mutant), so it runs +# nightly and on demand — never on pull requests — to avoid PR latency. It is +# informational/non-blocking: the score is reported to the run summary and the +# HTML report is uploaded as an artifact. + +on: + schedule: + - cron: "0 4 * * *" # nightly, 04:00 UTC + workflow_dispatch: + inputs: + mutate: + description: "Optional glob to scope mutation (e.g. src/tokens/**/*.ts)" + required: false + type: string + +concurrency: + group: mutation-${{ github.ref }} + cancel-in-progress: true + +jobs: + stryker: + name: Stryker (packages/pipeline) + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: 9 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run Stryker mutation testing + working-directory: packages/pipeline + run: | + # Run from packages/pipeline so the pnpm-linked node_modules resolve in + # the Stryker sandbox; the config lives at the repo root. + if [ -n "${{ github.event.inputs.mutate }}" ]; then + npx stryker run ../../stryker.conf.json --mutate "${{ github.event.inputs.mutate }}" + else + npx stryker run ../../stryker.conf.json + fi + + - name: Mutation score summary + if: always() + working-directory: packages/pipeline + run: | + node -e ' + const fs = require("fs"); + const path = "reports/mutation/mutation-report.json"; + if (!fs.existsSync(path)) { console.log("No mutation report produced."); process.exit(0); } + const report = JSON.parse(fs.readFileSync(path, "utf-8")); + const counts = { Killed: 0, Survived: 0, Timeout: 0, NoCoverage: 0, RuntimeError: 0, CompileError: 0, Ignored: 0 }; + for (const file of Object.values(report.files || {})) { + for (const m of file.mutants || []) counts[m.status] = (counts[m.status] || 0) + 1; + } + const detected = counts.Killed + counts.Timeout; + const valid = detected + counts.Survived + counts.NoCoverage; + const score = valid ? ((detected / valid) * 100).toFixed(2) : "n/a"; + const out = [ + "## Mutation testing (packages/pipeline)", + "", + `**Mutation score: ${score}%**` + (score === "n/a" ? "" : ` (target 80%, opt-in)`), + "", + "| Status | Count |", + "| --- | --- |", + ...Object.entries(counts).map(([k, v]) => `| ${k} | ${v} |`), + ].join("\n"); + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY || "/dev/stdout", out + "\n"); + console.log(`Mutation score: ${score}%`); + ' + + - name: Upload mutation report + if: always() + uses: actions/upload-artifact@v4 + with: + name: mutation-report + path: packages/pipeline/reports/mutation/ + if-no-files-found: ignore + retention-days: 14 diff --git a/README.md b/README.md index 4a29f63..c34c9f5 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,33 @@ Full catalog: `.claude/skills/README.md` ./scripts/setup-playwright.sh # One-time browser engine setup ``` +### Mutation testing (Stryker) + +Mutation testing measures how well the test suite actually catches bugs. The +framework's own TypeScript code (`packages/pipeline`) is mutation-tested via +[Stryker](https://stryker-mutator.io/); `stryker.conf.json` lives at the repo +root but **runs from `packages/pipeline`** so the pnpm-linked dependencies resolve +inside Stryker's sandbox. + +```bash +# Full run (slow — re-runs the suite per mutant) +cd packages/pipeline && npx stryker run ../../stryker.conf.json + +# Scope to one area while iterating +cd packages/pipeline && npx stryker run ../../stryker.conf.json --mutate "src/tokens/**/*.ts" +``` + +The HTML report is written to `packages/pipeline/reports/mutation/index.html`. + +- **Threshold (opt-in).** The target mutation score lives in + `.claude/pipeline.config.json` → `qualityGate.mutationScore` + (`{ "enabled": false, "threshold": 80, "blocking": false }`). It is **off by + default**; set `enabled: true` to treat it as a quality gate. +- **CI.** `.github/workflows/mutation.yml` runs Stryker **nightly** and + **on demand** (`workflow_dispatch`) — never on PRs, to avoid latency. It is + informational: the score is posted to the run summary and the HTML report is + uploaded as an artifact. + ### Pipeline Verification ```bash ./scripts/verify-tokens.sh # Catch hardcoded design values diff --git a/packages/pipeline/vitest.stryker.config.ts b/packages/pipeline/vitest.stryker.config.ts new file mode 100644 index 0000000..cccc125 --- /dev/null +++ b/packages/pipeline/vitest.stryker.config.ts @@ -0,0 +1,17 @@ +import { configDefaults, defineConfig } from "vitest/config"; + +/** + * Vitest config used by Stryker mutation runs (see ../../stryker.conf.json). + * + * Identical to vitest.config.ts except it excludes the end-to-end test, which + * reads a fixture at the repo root — outside the Stryker sandbox (cwd = + * packages/pipeline) — and would fail the initial test run. Unit tests provide + * the mutant-killing signal. + */ +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + environment: "node", + exclude: [...configDefaults.exclude, "**/__tests__/e2e.test.ts"], + }, +}); diff --git a/stryker.conf.json b/stryker.conf.json new file mode 100644 index 0000000..322e7db --- /dev/null +++ b/stryker.conf.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://raw.githubusercontent.com/stryker-mutator/stryker/master/packages/core/schema/stryker-schema.json", + "_comment": "Mutation testing for the framework's own TypeScript code in packages/pipeline. Run from packages/pipeline (see .github/workflows/mutation.yml and the README) so the pnpm-linked node_modules resolve inside the Stryker sandbox.", + "packageManager": "pnpm", + "plugins": ["@stryker-mutator/vitest-runner", "@stryker-mutator/typescript-checker"], + "testRunner": "vitest", + "vitest": { "configFile": "vitest.stryker.config.ts" }, + "checkers": ["typescript"], + "tsconfigFile": "tsconfig.json", + "mutate": [ + "src/**/*.ts", + "!src/**/*.test.ts", + "!src/**/__tests__/**", + "!src/**/*.d.ts", + "!src/**/index.ts", + "!src/**/cli.ts", + "!src/**/pipeline-cli.ts" + ], + "reporters": ["clear-text", "json", "html"], + "htmlReporter": { "fileName": "reports/mutation/index.html" }, + "jsonReporter": { "fileName": "reports/mutation/mutation-report.json" }, + "thresholds": { "high": 80, "low": 60, "break": null }, + "concurrency": 2, + "timeoutMS": 60000, + "tempDirName": ".stryker-tmp", + "cleanTempDir": true, + "ignoreStatic": true +}