From 45bc068b434c6a687164bc6f389b6bd36d3045c7 Mon Sep 17 00:00:00 2001 From: PAMulligan Date: Sat, 20 Jun 2026 06:02:15 -0400 Subject: [PATCH] ci(mutation): wire Stryker mutation testing into nightly CI Stryker deps were declared but no workflow ran them. Add a working root config, a nightly/on-demand workflow, and docs (#80). - stryker.conf.json: mutation-tests packages/pipeline (TS); runs from the package so pnpm-linked node_modules resolve in the sandbox; vitest runner + typescript checker; excludes barrels/CLIs/entry points - vitest.stryker.config.ts: excludes the e2e test (reads a repo-root fixture outside the sandbox) from mutation runs - .github/workflows/mutation.yml: nightly + workflow_dispatch (never on PRs); posts the score to the run summary, uploads the HTML report - README: how to run locally + the opt-in threshold (qualityGate.mutationScore in pipeline.config.json, enabled:false, threshold 80) - Verified locally: a complete scoped run finishes and produces a mutation score Refs #80 Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/mutation.yml | 90 ++++++++++++++++++++++ README.md | 27 +++++++ packages/pipeline/vitest.stryker.config.ts | 17 ++++ stryker.conf.json | 28 +++++++ 4 files changed, 162 insertions(+) create mode 100644 .github/workflows/mutation.yml create mode 100644 packages/pipeline/vitest.stryker.config.ts create mode 100644 stryker.conf.json diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml new file mode 100644 index 0000000..821ae29 --- /dev/null +++ b/.github/workflows/mutation.yml @@ -0,0 +1,90 @@ +name: Mutation Testing + +# Mutation testing is slow (it re-runs the test suite per mutant), so it runs +# nightly and on demand — never on pull requests — to avoid PR latency. It is +# informational/non-blocking: the score is reported to the run summary and the +# HTML report is uploaded as an artifact. + +on: + schedule: + - cron: "0 4 * * *" # nightly, 04:00 UTC + workflow_dispatch: + inputs: + mutate: + description: "Optional glob to scope mutation (e.g. src/tokens/**/*.ts)" + required: false + type: string + +concurrency: + group: mutation-${{ github.ref }} + cancel-in-progress: true + +jobs: + stryker: + name: Stryker (packages/pipeline) + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: 9 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run Stryker mutation testing + working-directory: packages/pipeline + run: | + # Run from packages/pipeline so the pnpm-linked node_modules resolve in + # the Stryker sandbox; the config lives at the repo root. + if [ -n "${{ github.event.inputs.mutate }}" ]; then + npx stryker run ../../stryker.conf.json --mutate "${{ github.event.inputs.mutate }}" + else + npx stryker run ../../stryker.conf.json + fi + + - name: Mutation score summary + if: always() + working-directory: packages/pipeline + run: | + node -e ' + const fs = require("fs"); + const path = "reports/mutation/mutation-report.json"; + if (!fs.existsSync(path)) { console.log("No mutation report produced."); process.exit(0); } + const report = JSON.parse(fs.readFileSync(path, "utf-8")); + const counts = { Killed: 0, Survived: 0, Timeout: 0, NoCoverage: 0, RuntimeError: 0, CompileError: 0, Ignored: 0 }; + for (const file of Object.values(report.files || {})) { + for (const m of file.mutants || []) counts[m.status] = (counts[m.status] || 0) + 1; + } + const detected = counts.Killed + counts.Timeout; + const valid = detected + counts.Survived + counts.NoCoverage; + const score = valid ? ((detected / valid) * 100).toFixed(2) : "n/a"; + const out = [ + "## Mutation testing (packages/pipeline)", + "", + `**Mutation score: ${score}%**` + (score === "n/a" ? "" : ` (target 80%, opt-in)`), + "", + "| Status | Count |", + "| --- | --- |", + ...Object.entries(counts).map(([k, v]) => `| ${k} | ${v} |`), + ].join("\n"); + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY || "/dev/stdout", out + "\n"); + console.log(`Mutation score: ${score}%`); + ' + + - name: Upload mutation report + if: always() + uses: actions/upload-artifact@v4 + with: + name: mutation-report + path: packages/pipeline/reports/mutation/ + if-no-files-found: ignore + retention-days: 14 diff --git a/README.md b/README.md index 4a29f63..c34c9f5 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,33 @@ Full catalog: `.claude/skills/README.md` ./scripts/setup-playwright.sh # One-time browser engine setup ``` +### Mutation testing (Stryker) + +Mutation testing measures how well the test suite actually catches bugs. The +framework's own TypeScript code (`packages/pipeline`) is mutation-tested via +[Stryker](https://stryker-mutator.io/); `stryker.conf.json` lives at the repo +root but **runs from `packages/pipeline`** so the pnpm-linked dependencies resolve +inside Stryker's sandbox. + +```bash +# Full run (slow — re-runs the suite per mutant) +cd packages/pipeline && npx stryker run ../../stryker.conf.json + +# Scope to one area while iterating +cd packages/pipeline && npx stryker run ../../stryker.conf.json --mutate "src/tokens/**/*.ts" +``` + +The HTML report is written to `packages/pipeline/reports/mutation/index.html`. + +- **Threshold (opt-in).** The target mutation score lives in + `.claude/pipeline.config.json` → `qualityGate.mutationScore` + (`{ "enabled": false, "threshold": 80, "blocking": false }`). It is **off by + default**; set `enabled: true` to treat it as a quality gate. +- **CI.** `.github/workflows/mutation.yml` runs Stryker **nightly** and + **on demand** (`workflow_dispatch`) — never on PRs, to avoid latency. It is + informational: the score is posted to the run summary and the HTML report is + uploaded as an artifact. + ### Pipeline Verification ```bash ./scripts/verify-tokens.sh # Catch hardcoded design values diff --git a/packages/pipeline/vitest.stryker.config.ts b/packages/pipeline/vitest.stryker.config.ts new file mode 100644 index 0000000..cccc125 --- /dev/null +++ b/packages/pipeline/vitest.stryker.config.ts @@ -0,0 +1,17 @@ +import { configDefaults, defineConfig } from "vitest/config"; + +/** + * Vitest config used by Stryker mutation runs (see ../../stryker.conf.json). + * + * Identical to vitest.config.ts except it excludes the end-to-end test, which + * reads a fixture at the repo root — outside the Stryker sandbox (cwd = + * packages/pipeline) — and would fail the initial test run. Unit tests provide + * the mutant-killing signal. + */ +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + environment: "node", + exclude: [...configDefaults.exclude, "**/__tests__/e2e.test.ts"], + }, +}); diff --git a/stryker.conf.json b/stryker.conf.json new file mode 100644 index 0000000..322e7db --- /dev/null +++ b/stryker.conf.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://raw.githubusercontent.com/stryker-mutator/stryker/master/packages/core/schema/stryker-schema.json", + "_comment": "Mutation testing for the framework's own TypeScript code in packages/pipeline. Run from packages/pipeline (see .github/workflows/mutation.yml and the README) so the pnpm-linked node_modules resolve inside the Stryker sandbox.", + "packageManager": "pnpm", + "plugins": ["@stryker-mutator/vitest-runner", "@stryker-mutator/typescript-checker"], + "testRunner": "vitest", + "vitest": { "configFile": "vitest.stryker.config.ts" }, + "checkers": ["typescript"], + "tsconfigFile": "tsconfig.json", + "mutate": [ + "src/**/*.ts", + "!src/**/*.test.ts", + "!src/**/__tests__/**", + "!src/**/*.d.ts", + "!src/**/index.ts", + "!src/**/cli.ts", + "!src/**/pipeline-cli.ts" + ], + "reporters": ["clear-text", "json", "html"], + "htmlReporter": { "fileName": "reports/mutation/index.html" }, + "jsonReporter": { "fileName": "reports/mutation/mutation-report.json" }, + "thresholds": { "high": 80, "low": 60, "break": null }, + "concurrency": 2, + "timeoutMS": 60000, + "tempDirName": ".stryker-tmp", + "cleanTempDir": true, + "ignoreStatic": true +}