diff --git a/.claude/commands/build-from-figma.md b/.claude/commands/build-from-figma.md index 5a28282..6bb82d9 100644 --- a/.claude/commands/build-from-figma.md +++ b/.claude/commands/build-from-figma.md @@ -278,21 +278,20 @@ Generate and run end-to-end tests appropriate to the app type. ## Phase 7: Cross-Browser Verification (Non-Blocking) -Capture screenshots in Firefox and WebKit, compare against Chromium baseline. +Diff Firefox and WebKit screenshots against their committed baselines with +provenance verification (RFC 0002). ```bash # Only for web apps and PWAs (Chrome extensions are Chromium-only) -./scripts/cross-browser-test.sh firefox http://localhost:3000 -./scripts/cross-browser-test.sh webkit http://localhost:3000 - -node scripts/visual-diff.js --batch \ - .claude/visual-qa/screenshots/firefox \ - .claude/visual-qa/screenshots/chromium \ - --output-dir .claude/visual-qa/diffs/firefox-vs-chromium \ - --threshold 0.03 +./scripts/cross-browser-baseline.sh compare http://localhost:3000 --json ``` -Cross-browser differences are logged in the build report but do NOT block the pipeline. +If no cross-browser baselines are committed yet the compare skips with a +capture hint (`./scripts/cross-browser-baseline.sh capture ` in the +pinned Playwright container, then commit `.claude/visual-qa/baselines/`). + +Cross-browser differences are logged in the build report but do NOT block the +pipeline (`visualBaselines.blocking: false`). ## Phase 8: Quality Gate diff --git a/.claude/pipeline.config.json b/.claude/pipeline.config.json index 43f3d23..1ec88a4 100644 --- a/.claude/pipeline.config.json +++ b/.claude/pipeline.config.json @@ -313,6 +313,41 @@ "browsers": ["chromium"], "reportFile": "regression-report.md" }, + "visualBaselines": { + "enabled": true, + "backend": "commit", + "storage": "git", + "baselineDir": ".claude/visual-qa/baselines", + "screenshotDir": ".claude/visual-qa/screenshots/cross-browser", + "diffDir": ".claude/visual-qa/diffs/cross-browser", + "browsers": ["chromium", "firefox", "webkit"], + "routes": ["/"], + "breakpoints": { + "mobile": 375, + "desktop": 1440 + }, + "threshold": 0.03, + "blocking": false, + "reportFile": "cross-browser-report.md", + "capture": { + "mode": "container", + "image": "mcr.microsoft.com/playwright:v1.61.1-noble", + "waitAfterLoadMs": 1500, + "fullPage": true + }, + "provenance": { + "manifest": ".claude/visual-qa/baselines/manifest.json", + "policy": "warn" + }, + "ciArtifact": { + "compareAgainst": "last-green-main", + "retentionDays": 30 + }, + "service": { + "provider": "chromatic", + "projectTokenEnv": "CHROMATIC_PROJECT_TOKEN" + } + }, "deadCode": { "enabled": true, "tool": "knip", @@ -472,9 +507,9 @@ }, "cross-browser": { "depends": ["component-build"], - "resources": [], + "resources": [{ "name": "port:dev-server", "mode": "shared" }], "blocking": false, - "description": "Firefox and WebKit screenshot comparison" + "description": "Firefox and WebKit baseline comparison (cross-browser-baseline.sh compare)" }, "quality-gate": { "depends": ["component-build"], diff --git a/.claude/pipeline.config.schema.json b/.claude/pipeline.config.schema.json index f3bfe41..e5f6a9b 100644 --- a/.claude/pipeline.config.schema.json +++ b/.claude/pipeline.config.schema.json @@ -362,6 +362,78 @@ } }, + "visualBaselines": { + "type": "object", + "additionalProperties": false, + "description": "Cross-browser (firefox/webkit) screenshot baseline storage per RFC 0002: pluggable backend, deterministic containerized capture, and per-baseline provenance. Same-browser chromium regression stays in regressionTesting.", + "properties": { + "enabled": { "type": "boolean" }, + "backend": { + "type": "string", + "enum": ["commit", "ci-artifact", "service"], + "description": "Where baselines live: committed to the repo (default), fetched from the last-green-main CI artifact, or delegated to a SaaS provider." + }, + "storage": { + "type": "string", + "enum": ["git", "lfs"], + "description": "commit backend only: plain git-tracked PNGs, or Git LFS for large baseline sets (see scripts/setup-baseline-lfs.sh)." + }, + "baselineDir": { "type": "string", "description": "Baseline root, shared with regressionTesting; per-engine subdirectories." }, + "screenshotDir": { "type": "string", "description": "Transient current-capture directory (gitignored)." }, + "diffDir": { "type": "string", "description": "Transient diff-image directory (gitignored)." }, + "browsers": { "$ref": "#/$defs/browserList" }, + "routes": { "type": "array", "items": { "type": "string" } }, + "breakpoints": { "$ref": "#/$defs/breakpointMap" }, + "threshold": { "$ref": "#/$defs/percentageRatio", "description": "Cross-engine mismatch tolerance. Must equal e2e.crossBrowserDiffThreshold (structural check)." }, + "blocking": { "type": "boolean", "description": "When true, compare failures exit non-zero (Phase B teeth). Keep false until determinism is proven." }, + "reportFile": { "type": "string" }, + "capture": { + "type": "object", + "additionalProperties": false, + "properties": { + "mode": { + "type": "string", + "enum": ["container", "local"], + "description": "container (required for trustworthy firefox/webkit baselines) runs capture in the pinned Playwright image; local captures are recorded with host=local and flagged by provenance." + }, + "image": { "type": "string", "description": "Pinned Playwright container image (authoritative; scripts warn on drift vs the resolved @playwright/test version)." }, + "waitAfterLoadMs": { "type": "integer", "minimum": 0 }, + "fullPage": { "type": "boolean" } + } + }, + "provenance": { + "type": "object", + "additionalProperties": false, + "properties": { + "manifest": { "type": "string", "description": "Committed provenance manifest path (engine, Playwright version, image, sha256, gitSha, host per baseline)." }, + "policy": { + "type": "string", + "enum": ["warn", "enforce"], + "description": "warn: provenance mismatches are reported but baselines are still diffed. enforce: flagged baselines are excluded from diffing and counted as failures." + } + } + }, + "ciArtifact": { + "type": "object", + "additionalProperties": false, + "description": "backend=ci-artifact only.", + "properties": { + "compareAgainst": { "type": "string", "enum": ["last-green-main"] }, + "retentionDays": { "type": "integer", "minimum": 1 } + } + }, + "service": { + "type": "object", + "additionalProperties": false, + "description": "backend=service only. The provider owns capture, diffing, and review; opt-in, requires a token in CI secrets.", + "properties": { + "provider": { "type": "string", "enum": ["chromatic", "percy"] }, + "projectTokenEnv": { "type": "string", "description": "Name of the environment variable holding the provider project token." } + } + } + } + }, + "deadCode": { "type": "object", "additionalProperties": false, diff --git a/.claude/skills/figma-to-react-workflow/SKILL.md b/.claude/skills/figma-to-react-workflow/SKILL.md index 659d18e..38211dc 100644 --- a/.claude/skills/figma-to-react-workflow/SKILL.md +++ b/.claude/skills/figma-to-react-workflow/SKILL.md @@ -480,22 +480,28 @@ For each page in build-spec: ### Step 3.2.5: Cross-Browser Verification -After Chromium passes the visual diff loop, verify in other browsers: +After Chromium passes the visual diff loop, diff Firefox and WebKit against +their committed baselines (RFC 0002): ```bash -# Capture screenshots in Firefox and WebKit -./scripts/cross-browser-test.sh firefox http://localhost:3000 -./scripts/cross-browser-test.sh webkit http://localhost:3000 - -# Compare against Chromium baseline (higher threshold for browser differences) -node scripts/visual-diff.js --batch \ - .claude/visual-qa/screenshots/firefox \ - .claude/visual-qa/screenshots/chromium \ - --output-dir .claude/visual-qa/diffs/firefox-vs-chromium \ - --threshold 0.03 +# Compare current firefox/webkit screenshots against committed baselines +# (threshold from visualBaselines.threshold, 0.03 for cross-engine tolerance) +./scripts/cross-browser-baseline.sh compare http://localhost:3000 --json ``` -Cross-browser results are logged in the build report but are **not blocking** by default. +If no cross-browser baselines exist yet, the compare skips with a hint — +capture them deterministically first and commit the result: + +```bash +./scripts/cross-browser-baseline.sh capture http://localhost:3000 # pinned Playwright container +git add .claude/visual-qa/baselines && git commit -m "test: add cross-browser baselines" +``` + +Each baseline's provenance (engine, Playwright version, container image, +sha256, host) is verified from `baselines/manifest.json` before diffing, so +stale or locally-captured baselines are flagged instead of producing false +pixel diffs. Results are logged in the build report but are **not blocking** +by default (`visualBaselines.blocking`). ### Step 3.3: E2E Tests @@ -686,7 +692,8 @@ This skill works with: - **scripts/visual-diff.js** — Pixel-level screenshot comparison with region analysis - **scripts/verify-tokens.sh** — Token integrity enforcement in quality gate - **scripts/verify-test-coverage.sh** — Test existence verification (TDD gate) -- **scripts/cross-browser-test.sh** — Multi-browser screenshot capture +- **scripts/cross-browser-baseline.sh** — Cross-browser baseline capture + provenance-verified comparison (RFC 0002) +- **scripts/cross-browser-test.sh** — Ad-hoc multi-browser screenshot capture (no comparison) - **scripts/generate-stories.sh** — AST-based Storybook story + MDX generation (Phase 4.5) - **.claude/pipeline.config.json** — Thresholds, iteration limits, app type config diff --git a/.claude/skills/parallel-orchestration/SKILL.md b/.claude/skills/parallel-orchestration/SKILL.md index 77e0663..b2be730 100644 --- a/.claude/skills/parallel-orchestration/SKILL.md +++ b/.claude/skills/parallel-orchestration/SKILL.md @@ -145,7 +145,7 @@ function hasResourceConflict(candidatePhase, runningPhases): | `visual-diff` | **Agent:** invoke visual-qa-verification skill with pixel-diff loop (max iterations from `iterationLoop.maxVisualIterations`) | | `dark-mode` | **Bash:** `./scripts/check-dark-mode.sh http://localhost:3000` | | `e2e-tests` | **Agent:** invoke e2e-test-generator skill with flows from build-spec.json | -| `cross-browser` | **Bash:** `./scripts/cross-browser-test.sh firefox http://localhost:3000 && ./scripts/cross-browser-test.sh webkit http://localhost:3000` | +| `cross-browser` | **Bash:** `./scripts/cross-browser-baseline.sh compare http://localhost:3000 --json` (diffs firefox/webkit against committed baselines with provenance checks — RFC 0002; skips with a capture hint when no baselines exist) | | `quality-gate` | **Parallel sub-dispatch:** run all `qualityGateSubtasks` concurrently (see below) | | `responsive` | **Bash:** `./scripts/check-responsive.sh http://localhost:3000` | | `report` | **Agent:** generate `build-report.md` in `.claude/visual-qa/` from all collected phase results, timings, and diff images | diff --git a/.claude/skills/visual-qa-verification/SKILL.md b/.claude/skills/visual-qa-verification/SKILL.md index c567ae8..0f764ca 100644 --- a/.claude/skills/visual-qa-verification/SKILL.md +++ b/.claude/skills/visual-qa-verification/SKILL.md @@ -143,35 +143,28 @@ When the diff reports failing regions, use this mapping to prioritize fixes: ### Step 2.5: Cross-Browser Verification -After Chromium passes the visual diff loop, verify in Firefox and WebKit. - -**Using Playwright MCP or cross-browser-test.sh:** +After Chromium passes the visual diff loop, diff Firefox and WebKit against +their committed same-engine baselines (RFC 0002): ```bash -# Capture Firefox screenshots -./scripts/cross-browser-test.sh firefox http://localhost:3000 - -# Capture WebKit screenshots -./scripts/cross-browser-test.sh webkit http://localhost:3000 - -# Compare Firefox against Chromium baseline (not Figma) -node scripts/visual-diff.js --batch \ - .claude/visual-qa/screenshots/firefox \ - .claude/visual-qa/screenshots/chromium \ - --output-dir .claude/visual-qa/diffs/firefox-vs-chromium \ - --threshold 0.03 - -# Compare WebKit against Chromium baseline -node scripts/visual-diff.js --batch \ - .claude/visual-qa/screenshots/webkit \ - .claude/visual-qa/screenshots/chromium \ - --output-dir .claude/visual-qa/diffs/webkit-vs-chromium \ - --threshold 0.03 +# Compare current firefox/webkit screenshots against committed baselines, +# with per-baseline provenance verification from baselines/manifest.json +./scripts/cross-browser-baseline.sh compare http://localhost:3000 --json + +# First time (no baselines committed yet): capture deterministically in the +# pinned Playwright container, then commit +./scripts/cross-browser-baseline.sh capture http://localhost:3000 +git add .claude/visual-qa/baselines + +# Provenance-only check (no dev server needed) +./scripts/cross-browser-baseline.sh verify ``` -**Cross-browser threshold:** Use a slightly higher threshold (3% vs 2%) since browser rendering engines have legitimate differences in anti-aliasing, font rendering, and sub-pixel positioning. +For ad-hoc engine captures without comparison, `./scripts/cross-browser-test.sh ` still works. + +**Cross-browser threshold:** `visualBaselines.threshold` (3% vs the 2% same-browser threshold) since browser rendering engines have legitimate differences in anti-aliasing, font rendering, and sub-pixel positioning. -**Cross-browser failures are NOT blocking** by default (configurable in `pipeline.config.json` → `qualityGate.crossBrowserScreenshotsRequired`). They are reported in the build report for manual review. +**Cross-browser failures are NOT blocking** by default (`visualBaselines.blocking`, and `qualityGate.crossBrowserScreenshotsRequired` for the gate). They are reported in the build report for manual review. Compares run outside the pinned container are marked *advisory* by the provenance check. **Common cross-browser differences to ignore:** - Font rendering (especially on macOS WebKit vs Windows Chromium) @@ -349,7 +342,8 @@ This skill works with: - **Playwright MCP** -- Cross-browser screenshots (Firefox, WebKit) and automated interaction testing - **Figma MCP** -- Source design screenshots for comparison (`get_screenshot`, `get_design_context`) - **scripts/visual-diff.js** -- Pixel-level screenshot comparison with region analysis -- **scripts/cross-browser-test.sh** -- Multi-browser screenshot capture +- **scripts/cross-browser-baseline.sh** -- Cross-browser baseline capture + provenance-verified comparison (RFC 0002) +- **scripts/cross-browser-test.sh** -- Ad-hoc multi-browser screenshot capture (no comparison) - **.claude/pipeline.config.json** -- Thresholds, iteration limits, breakpoint configuration ## Verification Report Template diff --git a/.github/workflows/cross-browser-baselines.yml b/.github/workflows/cross-browser-baselines.yml new file mode 100644 index 0000000..d0b1bcb --- /dev/null +++ b/.github/workflows/cross-browser-baselines.yml @@ -0,0 +1,329 @@ +name: Cross-Browser Baselines + +# RFC 0002 — cross-browser screenshot baselines. +# +# pull_request → compare current firefox/webkit/chromium screenshots +# against stored baselines inside the pinned Playwright +# container (deterministic envelope, provenance-checked) +# workflow_dispatch → recapture baselines deterministically and open a PR +# with the refreshed PNGs + provenance manifest +# push to main → when visualBaselines.backend is "ci-artifact", +# capture and publish the baselines artifact that PRs +# compare against (last-green-main) +# +# The pinned image comes from pipeline.config.json → visualBaselines.capture.image +# (read by the prep job), so the config stays the single source of truth. + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +concurrency: + group: cross-browser-${{ github.ref }} + cancel-in-progress: true + +jobs: + # ── Read visualBaselines config on a bare runner ────────────────────── + prep: + name: Read Config + runs-on: ubuntu-latest + timeout-minutes: 2 + outputs: + enabled: ${{ steps.config.outputs.enabled }} + image: ${{ steps.config.outputs.image }} + backend: ${{ steps.config.outputs.backend }} + pwversion: ${{ steps.config.outputs.pwversion }} + retention: ${{ steps.config.outputs.retention }} + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Extract visualBaselines settings + id: config + run: | + get() { node scripts/lib/pipeline-config.js get "$1" "$2"; } + IMAGE=$(get visualBaselines.capture.image mcr.microsoft.com/playwright:v1.61.1-noble) + { + echo "enabled=$(get visualBaselines.enabled true)" + echo "image=$IMAGE" + echo "backend=$(get visualBaselines.backend commit)" + echo "retention=$(get visualBaselines.ciArtifact.retentionDays 30)" + echo "pwversion=$(echo "$IMAGE" | sed -n 's/.*:v\([0-9.]*\)-.*/\1/p')" + } >> "$GITHUB_OUTPUT" + + # ── Compare against stored baselines inside the pinned image (PR) ───── + compare: + name: Compare (pinned container) + runs-on: ubuntu-latest + needs: prep + timeout-minutes: 10 + if: github.event_name == 'pull_request' && needs.prep.outputs.enabled == 'true' + container: + image: ${{ needs.prep.outputs.image }} + steps: + - uses: actions/checkout@v4 + + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: 9 + + - name: Check for cross-browser baselines + id: baselines + run: | + COUNT=0 + for engine in chromium firefox webkit; do + C=$(find ".claude/visual-qa/baselines/$engine" -name "*.png" 2>/dev/null | wc -l) + COUNT=$((COUNT + C)) + done + echo "count=$COUNT" >> "$GITHUB_OUTPUT" + if [ "$COUNT" -eq 0 ]; then + echo "No cross-browser baselines found — skipping compare" + else + echo "Found $COUNT baseline screenshots" + fi + + - name: Install repo dependencies + if: steps.baselines.outputs.count != '0' + run: pnpm install --frozen-lockfile + + - name: Install @playwright/test at the image version + if: steps.baselines.outputs.count != '0' + run: | + mkdir -p /tmp/cbb && cd /tmp/cbb + npm init -y >/dev/null 2>&1 + npm i --no-fund --no-audit "@playwright/test@${{ needs.prep.outputs.pwversion }}" + + - name: Install app dependencies + if: steps.baselines.outputs.count != '0' && hashFiles('app/package.json') != '' + working-directory: app + run: pnpm install --frozen-lockfile + + - name: Build app + if: steps.baselines.outputs.count != '0' && hashFiles('app/package.json') != '' + working-directory: app + run: pnpm build + + - name: Start app server + if: steps.baselines.outputs.count != '0' && hashFiles('app/package.json') != '' + working-directory: app + run: | + pnpm start & + sleep 5 + + - name: Compare against baselines + if: steps.baselines.outputs.count != '0' && hashFiles('app/package.json') != '' + env: + CBB_PLAYWRIGHT_DIR: /tmp/cbb + CBB_IN_CONTAINER: "1" + # Exit code honors visualBaselines.blocking: non-blocking runs exit 0 + # and surface results in the report/PR comment (RFC 0002 Phase A→B). + run: node scripts/cross-browser-baseline.js compare http://localhost:3000 --json + + - name: Upload diff artifacts + if: steps.baselines.outputs.count != '0' && always() + uses: actions/upload-artifact@v4 + with: + name: cross-browser-diffs + path: | + .claude/visual-qa/diffs/cross-browser/ + .claude/visual-qa/cross-browser-report.md + retention-days: 14 + if-no-files-found: ignore + + - name: Comment PR with cross-browser results + if: steps.baselines.outputs.count != '0' && always() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const reportPath = '.claude/visual-qa/cross-browser-report.md'; + let body = '## Cross-Browser Baseline Results\n\nNo report generated.'; + if (fs.existsSync(reportPath)) { + body = '## Cross-Browser Baseline Results\n\n' + fs.readFileSync(reportPath, 'utf-8'); + } + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const existing = comments.find(c => + c.user.type === 'Bot' && c.body.includes('Cross-Browser Baseline Results') + ); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + + # ── Recapture baselines deterministically and open a PR (manual) ────── + capture: + name: Capture & PR refreshed baselines + runs-on: ubuntu-latest + needs: prep + timeout-minutes: 15 + if: github.event_name == 'workflow_dispatch' && needs.prep.outputs.enabled == 'true' + container: + image: ${{ needs.prep.outputs.image }} + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: 9 + + - name: Mark workspace safe for git in-container + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Install repo dependencies + run: pnpm install --frozen-lockfile + + - name: Install @playwright/test at the image version + run: | + mkdir -p /tmp/cbb && cd /tmp/cbb + npm init -y >/dev/null 2>&1 + npm i --no-fund --no-audit "@playwright/test@${{ needs.prep.outputs.pwversion }}" + + - name: Install app dependencies + if: hashFiles('app/package.json') != '' + working-directory: app + run: pnpm install --frozen-lockfile + + - name: Build app + if: hashFiles('app/package.json') != '' + working-directory: app + run: pnpm build + + - name: Start app server + if: hashFiles('app/package.json') != '' + working-directory: app + run: | + pnpm start & + sleep 5 + + - name: Capture baselines in the pinned container + if: hashFiles('app/package.json') != '' + env: + CBB_PLAYWRIGHT_DIR: /tmp/cbb + CBB_IN_CONTAINER: "1" + run: node scripts/cross-browser-baseline.js capture http://localhost:3000 + + - name: Push refreshed baselines branch + if: hashFiles('app/package.json') != '' + id: push + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + BRANCH="chore/cross-browser-baselines-${GITHUB_RUN_ID}" + git checkout -b "$BRANCH" + git add .claude/visual-qa/baselines + if git diff --cached --quiet; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "Baselines unchanged — nothing to PR" + exit 0 + fi + git commit -m "test: refresh cross-browser baselines (pinned container capture)" + git push origin "$BRANCH" + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "branch=$BRANCH" >> "$GITHUB_OUTPUT" + + - name: Open baselines PR + if: steps.push.outputs.changed == 'true' + uses: actions/github-script@v7 + with: + script: | + await github.rest.pulls.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: 'test: refresh cross-browser baselines', + head: '${{ steps.push.outputs.branch }}', + base: 'main', + body: [ + 'Deterministic recapture of the cross-browser baselines inside the pinned', + 'Playwright container (`${{ needs.prep.outputs.image }}`), including the', + 'refreshed provenance manifest. Review the PNG diffs before merging (RFC 0002).', + ].join('\n'), + }); + + # ── Publish the baselines artifact from main (ci-artifact backend) ──── + publish: + name: Publish baselines artifact + runs-on: ubuntu-latest + needs: prep + timeout-minutes: 15 + if: github.event_name == 'push' && needs.prep.outputs.enabled == 'true' && needs.prep.outputs.backend == 'ci-artifact' + container: + image: ${{ needs.prep.outputs.image }} + steps: + - uses: actions/checkout@v4 + + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: 9 + + - name: Mark workspace safe for git in-container + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Install repo dependencies + run: pnpm install --frozen-lockfile + + - name: Install @playwright/test at the image version + run: | + mkdir -p /tmp/cbb && cd /tmp/cbb + npm init -y >/dev/null 2>&1 + npm i --no-fund --no-audit "@playwright/test@${{ needs.prep.outputs.pwversion }}" + + - name: Install app dependencies + if: hashFiles('app/package.json') != '' + working-directory: app + run: pnpm install --frozen-lockfile + + - name: Build app + if: hashFiles('app/package.json') != '' + working-directory: app + run: pnpm build + + - name: Start app server + if: hashFiles('app/package.json') != '' + working-directory: app + run: | + pnpm start & + sleep 5 + + - name: Capture baselines in the pinned container + if: hashFiles('app/package.json') != '' + env: + CBB_PLAYWRIGHT_DIR: /tmp/cbb + CBB_IN_CONTAINER: "1" + run: node scripts/cross-browser-baseline.js capture http://localhost:3000 + + - name: Upload baselines artifact (last-green-main) + if: hashFiles('app/package.json') != '' + uses: actions/upload-artifact@v4 + with: + name: cross-browser-baselines + path: .claude/visual-qa/baselines/ + retention-days: ${{ fromJSON(needs.prep.outputs.retention) }} + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 278ada0..6570f93 100644 --- a/.gitignore +++ b/.gitignore @@ -112,6 +112,9 @@ experiments/ .claude/visual-qa/screenshots/regression/**/*.png .claude/visual-qa/diffs/regression/**/*.png .claude/visual-qa/regression-report.md +.claude/visual-qa/screenshots/cross-browser/**/*.png +.claude/visual-qa/diffs/cross-browser/**/*.png +.claude/visual-qa/cross-browser-report.md # Taskmaster Files .taskmaster/tasks/*.json.bak diff --git a/CLAUDE.md b/CLAUDE.md index fb444a4..6150973 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,11 +74,19 @@ node scripts/visual-diff.js --batch [--output-dir di # Initialize a new React project ./scripts/setup-project.sh my-app --next # or --vite -# Cross-browser testing (Playwright) +# Cross-browser testing (Playwright, ad-hoc capture only) ./scripts/cross-browser-test.sh chromium http://localhost:3000 ./scripts/cross-browser-test.sh firefox http://localhost:3000 ./scripts/cross-browser-test.sh webkit http://localhost:3000 +# Cross-browser baselines: pinned-container capture, provenance-verified compare (RFC 0002) +./scripts/cross-browser-baseline.sh capture http://localhost:3000 [--local] [--dry-run] +./scripts/cross-browser-baseline.sh compare http://localhost:3000 [--json] [--blocking] +./scripts/cross-browser-baseline.sh verify [--json] + +# Route large baseline sets through Git LFS (visualBaselines.storage = "lfs") +./scripts/setup-baseline-lfs.sh [--dry-run] [--force] + # Setup Playwright browsers (one-time) ./scripts/setup-playwright.sh @@ -292,7 +300,7 @@ Autonomous 9-phase pipeline that converts a Figma design into a working, tested ├─ [5] VISUAL DIFF → pixelmatch loop → max 5 iterations │ └─ [6] E2E TESTS → e2e-test-generator skill ├─ [5.5] DARK MODE → check-dark-mode.sh (non-blocking) - ├─ [7] CROSS-BROWSER → Firefox/WebKit screenshots (non-blocking) + ├─ [7] CROSS-BROWSER → cross-browser-baseline.sh compare (non-blocking) ├─ [7.5] REGRESSION → regression-test.sh (non-blocking) ├─ [8] QUALITY GATE → [coverage|types|build|tokens|lighthouse] in parallel └─ [8.5] RESPONSIVE → check-responsive.sh (non-blocking) @@ -306,6 +314,7 @@ Autonomous 9-phase pipeline that converts a Figma design into a working, tested - `verify-tokens.sh` — Catches hardcoded values and token drift - `verify-test-coverage.sh` — Ensures every component has tests - `visual-diff.js` — Pixel-level screenshot comparison with region analysis +- `.claude/visual-qa/baselines/manifest.json` — Cross-browser baseline provenance (engine, Playwright version, pinned image, sha256, host — RFC 0002) - `sync-tokens.sh` — Detects token drift between lockfile and source - `check-dark-mode.sh` — Dark mode screenshot capture and visual comparison - `generate-stories.sh` — AST-based Storybook story + MDX generation with prop controls, variants, and action args @@ -317,7 +326,7 @@ Autonomous 9-phase pipeline that converts a Figma design into a working, tested - **App-type awareness** — Chrome extensions, PWAs, React Native, and web apps get tailored E2E strategies - **Chrome extension E2E** — Playwright persistent context with `--load-extension` - Design token extraction with lockfile enforcement -- Cross-browser verification (Firefox, WebKit) with configurable thresholds +- Cross-browser verification (Firefox, WebKit) — committed per-engine baselines diffed at the 0.03 cross-engine threshold, with pinned-container capture and per-baseline provenance (RFC 0002; backends: commit | ci-artifact | service) - Quality gate: 80%+ coverage, TypeScript, Lighthouse audit - Resumable: TodoWrite tracks progress across interrupted sessions - **Dark mode verification** — automated dark theme screenshot comparison (non-blocking) @@ -586,6 +595,8 @@ gh issue create # Create issue ./scripts/audit-cross-browser-css.sh # Cross-browser CSS audit ./scripts/capture-baselines.sh # Capture regression baselines ./scripts/regression-test.sh # Visual regression testing +./scripts/cross-browser-baseline.sh # Cross-browser baseline capture/compare (RFC 0002) +./scripts/setup-baseline-lfs.sh # Git LFS opt-in for large baseline sets ./scripts/export-design-system.sh # Export components + tokens as pnpm workspace ./scripts/import-design-tokens.sh # Reconstruct lockfile from an export (round-trip / consumer) ./scripts/validate-pipeline-config.sh # Validate pipeline.config.json against schema @@ -623,6 +634,6 @@ node scripts/metrics-dashboard.js summary # Quick metrics summary --- **Last Updated:** 2026-07-01 -**Architecture:** 56 agents, 24 skills, 4 plugins + gh CLI, Figma + Canva + Playwright MCP, 40 scripts, 8 hooks, 5 renderers (nextjs, vite, astro, sveltekit, expo) +**Architecture:** 56 agents, 24 skills, 4 plugins + gh CLI, Figma + Canva + Playwright MCP, 55 scripts, 8 hooks, 5 renderers (nextjs, vite, astro, sveltekit, expo) > **Keeping counts in sync:** When adding or removing agents, skills, scripts, or hooks, update all count references across the project. Search for the old count number in `*.md` files to find all references: `CLAUDE.md`, `README.md`, `CONTRIBUTING.md`, `docs/onboarding/`, `docs/react-development/`, and `.claude/AGENT-NAMING-GUIDE.md`. The agent and skill counts are enforced automatically by `scripts/check-doc-counts.sh` (run in CI and on pre-commit), which recounts `.claude/agents/` and `.claude/skills/` and fails on any documented count that disagrees. diff --git a/README.md b/README.md index ba9b873..7fa179e 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,9 @@ project-root/ │ ├── verify-tokens.sh # Design token enforcement │ ├── verify-test-coverage.sh # Component test verification │ ├── visual-diff.js # Pixel-level screenshot comparison -│ ├── cross-browser-test.sh # Playwright multi-browser +│ ├── cross-browser-test.sh # Playwright multi-browser (ad-hoc capture) +│ ├── cross-browser-baseline.sh # Cross-browser baselines + provenance (RFC 0002) +│ ├── setup-baseline-lfs.sh # Git LFS opt-in for large baseline sets │ ├── setup-playwright.sh # One-time browser setup │ ├── check-dead-code.sh # Dead code detection (knip) │ ├── check-security.sh # Dependency audit + anti-patterns @@ -249,7 +251,9 @@ Full catalog: `.claude/skills/README.md` ### Testing ```bash ./scripts/run-tests.sh # Vitest unit/component tests -./scripts/cross-browser-test.sh # Playwright multi-browser screenshots +./scripts/cross-browser-test.sh # Playwright multi-browser screenshots (ad-hoc) +./scripts/cross-browser-baseline.sh # Cross-browser baseline capture/compare (RFC 0002) +./scripts/setup-baseline-lfs.sh # Route large baseline sets through Git LFS ./scripts/setup-playwright.sh # One-time browser engine setup ``` diff --git a/docs/guides/error-recovery.md b/docs/guides/error-recovery.md index 93faaa3..116395e 100644 --- a/docs/guides/error-recovery.md +++ b/docs/guides/error-recovery.md @@ -27,7 +27,7 @@ Every phase has characteristic failure patterns. The table below lists the most | 5: Visual Diff | Screenshot capture fails, diff threshold exceeded after 5 iterations | Check dev server is running at expected port, lower threshold or fix components | | 5.5: Dark Mode | Dark theme not configured, screenshot fails | Non-blocking — add dark mode support or skip | | 6: E2E Tests | Browser not installed, test timeout, element not found | Run `./scripts/setup-playwright.sh`, increase timeout in `pipeline.config.json` | -| 7: Cross-Browser | Firefox/WebKit not installed | Run `./scripts/setup-playwright.sh` to install all browsers | +| 7: Cross-Browser | No committed baselines (compare skips), provenance flags, Firefox/WebKit not installed | Capture + commit baselines via `./scripts/cross-browser-baseline.sh capture` (pinned container); `./scripts/setup-playwright.sh` for local engines; see `docs/regression-testing/cross-browser.md` | | 7.5: Regression | No baselines captured yet | Run `./scripts/capture-baselines.sh` first, then `./scripts/regression-test.sh` | | 8: Quality Gate | Coverage below 80%, TypeScript errors, Lighthouse below thresholds | Write more tests, fix type errors, optimize performance | | 8.5: Responsive | Dev server not running, screenshot timeout | Start dev server first, check ports | diff --git a/docs/onboarding/architecture.md b/docs/onboarding/architecture.md index 7ae6621..7483d49 100644 --- a/docs/onboarding/architecture.md +++ b/docs/onboarding/architecture.md @@ -291,7 +291,9 @@ Located in `scripts/`. These are the automation backbone that agents and pipelin | Script | Purpose | |--------|---------| | `run-tests.sh` | Vitest with coverage | -| `cross-browser-test.sh` | Playwright multi-browser screenshots | +| `cross-browser-test.sh` | Playwright multi-browser screenshots (ad-hoc capture, no comparison) | +| `cross-browser-baseline.sh` | Cross-browser baseline capture/compare with provenance (RFC 0002) | +| `setup-baseline-lfs.sh` | Route large baseline sets through Git LFS (storage="lfs") | | `setup-playwright.sh` | One-time browser engine installation | | `capture-baselines.sh` | Capture baseline screenshots for regression | | `regression-test.sh` | Visual regression testing against baselines | diff --git a/docs/onboarding/pipeline-configuration.md b/docs/onboarding/pipeline-configuration.md index b5f1471..1c7e52a 100644 --- a/docs/onboarding/pipeline-configuration.md +++ b/docs/onboarding/pipeline-configuration.md @@ -785,6 +785,71 @@ Visual regression testing against stored baselines. --- +## Cross-Browser Baselines (`visualBaselines`) + +Cross-browser (firefox/webkit) screenshot baseline storage per +[RFC 0002](../rfcs/0002-cross-browser-screenshot-baseline-storage.md): +pluggable backend, deterministic pinned-container capture, and per-baseline +provenance. Shares `baselineDir` with `regressionTesting` (each flow walks +only its own engines). Full guide: +[docs/regression-testing/cross-browser.md](../regression-testing/cross-browser.md). + +| Field | Type | Default | Description | +| ------------------------- | ------------------------------------------------ | ---------------------------------------------- | --------------------------------------------------------------------------------------------- | +| `enabled` | `boolean` | `true` | Enable the cross-browser baseline flow. | +| `backend` | `string` (enum: `commit`, `ci-artifact`, `service`) | `"commit"` | Where baselines live: git-tracked, last-green-main CI artifact, or a SaaS provider. | +| `storage` | `string` (enum: `git`, `lfs`) | `"git"` | commit backend only — `lfs` routes PNGs through Git LFS (`setup-baseline-lfs.sh`). | +| `baselineDir` | `string` | `".claude/visual-qa/baselines"` | Baseline root, shared with `regressionTesting`. | +| `screenshotDir` | `string` | `".claude/visual-qa/screenshots/cross-browser"`| Transient current captures (gitignored). | +| `diffDir` | `string` | `".claude/visual-qa/diffs/cross-browser"` | Transient diff images (gitignored). | +| `browsers` | `browserList` | `["chromium","firefox","webkit"]` | Engines to baseline; must stay within `e2e.crossBrowserBrowsers` (structural check). | +| `routes` | `string[]` | `["/"]` | Routes to capture. | +| `breakpoints` | `breakpointMap` | `{ "mobile": 375, "desktop": 1440 }` | Viewport name → pixel width. | +| `threshold` | `percentageRatio` (0–1) | `0.03` | Cross-engine tolerance; must equal `e2e.crossBrowserDiffThreshold` (structural check). | +| `blocking` | `boolean` | `false` | When `true`, compare exits non-zero on failures (Phase B teeth). | +| `reportFile` | `string` | `"cross-browser-report.md"` | Report filename under `.claude/visual-qa/`. | +| `capture.mode` | `string` (enum: `container`, `local`) | `"container"` | Container is required for trustworthy firefox/webkit baselines; local is recorded + flagged. | +| `capture.image` | `string` | `"mcr.microsoft.com/playwright:v1.61.1-noble"` | The determinism pin; scripts warn on drift vs the resolved Playwright version. | +| `capture.waitAfterLoadMs` | `integer (≥0)` | `1500` | Wait time before capture, in ms. | +| `capture.fullPage` | `boolean` | `true` | Capture full page. | +| `provenance.manifest` | `string` | `".claude/visual-qa/baselines/manifest.json"` | Committed provenance manifest (sha256, engine, image, host, gitSha per baseline). | +| `provenance.policy` | `string` (enum: `warn`, `enforce`) | `"warn"` | `enforce` excludes flagged baselines from diffing and counts them as failures. | +| `ciArtifact.compareAgainst` | `string` (enum: `last-green-main`) | `"last-green-main"` | ci-artifact backend: which build's artifact PRs compare against. | +| `ciArtifact.retentionDays` | `integer (≥1)` | `30` | Retention of the published baselines artifact. | +| `service.provider` | `string` (enum: `chromatic`, `percy`) | `"chromatic"` | service backend: which provider owns capture/diff/review. | +| `service.projectTokenEnv` | `string` | `"CHROMATIC_PROJECT_TOKEN"` | Env var holding the provider project token (set as a CI secret). | + +```json +"visualBaselines": { + "enabled": true, + "backend": "commit", + "storage": "git", + "baselineDir": ".claude/visual-qa/baselines", + "screenshotDir": ".claude/visual-qa/screenshots/cross-browser", + "diffDir": ".claude/visual-qa/diffs/cross-browser", + "browsers": ["chromium", "firefox", "webkit"], + "routes": ["/"], + "breakpoints": { "mobile": 375, "desktop": 1440 }, + "threshold": 0.03, + "blocking": false, + "reportFile": "cross-browser-report.md", + "capture": { + "mode": "container", + "image": "mcr.microsoft.com/playwright:v1.61.1-noble", + "waitAfterLoadMs": 1500, + "fullPage": true + }, + "provenance": { + "manifest": ".claude/visual-qa/baselines/manifest.json", + "policy": "warn" + }, + "ciArtifact": { "compareAgainst": "last-green-main", "retentionDays": 30 }, + "service": { "provider": "chromatic", "projectTokenEnv": "CHROMATIC_PROJECT_TOKEN" } +} +``` + +--- + ## Dead Code (`deadCode`) Unused export/dependency/file detection. diff --git a/docs/plans/2026-07-01-cross-browser-baseline-storage.md b/docs/plans/2026-07-01-cross-browser-baseline-storage.md new file mode 100644 index 0000000..e080011 --- /dev/null +++ b/docs/plans/2026-07-01-cross-browser-baseline-storage.md @@ -0,0 +1,154 @@ +# Cross-Browser Screenshot Baseline Storage (RFC 0002) Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Implement RFC 0002 — commit-backed, provenance-verified cross-browser (firefox/webkit) screenshot baselines with a pluggable backend abstraction (`commit` | `ci-artifact` | `service`), pinned-container capture, and Git LFS opt-in — closing issue #111. + +**Architecture:** A new `visualBaselines` section in `pipeline.config.json` (strict JSON Schema) drives a new `scripts/cross-browser-baseline.js` CLI (thin `scripts/cross-browser-baseline.sh` wrapper) with `capture` / `compare` / `verify` subcommands. Capture uses Playwright (resolved dynamically — it is not a repo dependency) and, for authoritative firefox/webkit baselines, runs inside the pinned `mcr.microsoft.com/playwright` container (docker locally, `container:` in CI). A committed `baselines/manifest.json` records per-baseline provenance (sha256, engine, playwrightVersion, image, host, gitSha) via a new dual-mode lib `scripts/lib/baseline-manifest.js`; the comparator verifies provenance and flags stale/foreign baselines instead of emitting false pixel diffs. Backends live behind a uniform adapter contract in `scripts/lib/baseline-backends.js`. Diffing reuses `scripts/visual-diff.js` (pixelmatch) at threshold 0.03. Non-blocking by default (`blocking: false`, `qualityGate.crossBrowserScreenshotsRequired: false` unchanged) per RFC §10 Phase A/B. + +**Tech Stack:** Node 22 ESM (`"type": "module"`), Bash + `scripts/lib/common.sh`, Playwright (dynamic resolution via `createRequire`), pixelmatch/pngjs (via existing `visual-diff.js`), ajv 2020-12, vitest (`scripts/__tests__/vitest.config.js`, black-box CLI tests), GitHub Actions, Docker (`mcr.microsoft.com/playwright:v1.61.1-noble`), Git LFS. + +--- + +## Design decisions (resolving RFC §13 open questions) + +These are recorded here and in `docs/regression-testing/cross-browser.md` (Task 10); the RFC itself only gets its Status/§12 rows updated per its own §12 instruction. + +1. **Unify `regressionTesting` and `visualBaselines`?** Keep separate (RFC §7.1 already states this). They share `baselineDir` and the per-engine directory layout; `regressionTesting` owns same-browser chromium at 0.02, `visualBaselines` owns the cross-engine dimension at 0.03. To avoid cross-noise, `regression-test.sh`'s baseline walk is restricted to *its* configured `browsers` (Task 5) so committed firefox/webkit baselines don't produce SKIP rows in the regression report. +2. **Blocking threshold (Phase B "teeth").** Machinery ships now: `visualBaselines.blocking` and `provenance.policy: "enforce"` are honored by the comparator. The flip itself stays manual and data-driven — documented guidance: flip after ≥20 consecutive containerized CI compare runs with 0 provenance flags and <5% of runs showing pixel failures not tied to an intentional UI change. `qualityGate.crossBrowserScreenshotsRequired` stays `false`. +3. **Image pinning strategy.** The config pin (`visualBaselines.capture.image`) is authoritative; scripts *warn* when the image tag's embedded version disagrees with the resolved `@playwright/test` version or the manifest's recorded version (drift is surfaced, never silently ignored). Pinned initially to `v1.61.1-noble` (the Playwright version on the capture host today). +4. **Local firefox/webkit capture.** Warn-and-record, not hard-fail: local captures are written with `host: "local"` in the manifest and the comparator flags them (`foreign-host`) per `provenance.policy`. Local *compares* against container baselines are likewise marked advisory (envelope mismatch). Only `policy: "enforce"` turns provenance flags into failures. +5. **LFS migration.** `scripts/setup-baseline-lfs.sh` automates forward-only adoption (`.gitattributes` filter + `git lfs install --local`); history rewrite (`git lfs migrate import`) is printed as documented guidance, never executed by the script. + +Additive fields beyond the RFC §7.1 sketch (needed operationally, same spirit): `screenshotDir`, `diffDir`, `reportFile` (mirroring `regressionTesting`), and `provenance.policy` (RFC §5: "refuses (or warns, per policy)"). + +## Contracts + +**Baseline layout (shared with regression testing):** `.claude/visual-qa/baselines///_px.png`, `routeSlug = '/' → 'home'`, else strip leading `/`, `/`→`-`. `manifest.json` sits at the baseline root (RFC §7.2). + +**Manifest (RFC §7.3):** +```json +{ + "playwrightVersion": "1.61.1", + "image": "mcr.microsoft.com/playwright:v1.61.1-noble", + "baselines": { + "firefox/home/desktop_1440px.png": { + "sha256": "…", "capturedAt": "2026-07-01T12:00:00Z", "gitSha": "abc1234", + "engine": "firefox", "route": "/", "breakpoint": "desktop", "host": "container" + } + } +} +``` + +**Provenance statuses (per baseline, from `verifyBaselines`):** `ok` | `untracked` (file without manifest entry) | `modified` (sha256 mismatch — rewritten outside capture) | `missing-file` (entry without file) | `foreign-host` (firefox/webkit entry with `host: "local"`) | `version-drift` (manifest `playwrightVersion` ≠ current envelope's) | `image-drift` (manifest `image` ≠ configured image). Comparison-level: `envelopeMatch: false` when the *current* capture host/version differs from the manifest envelope → results marked advisory. `policy: "warn"` → flags are warnings, pixel diff still runs; `policy: "enforce"` → flagged baselines are excluded from pixel diffing and counted as provenance failures. + +**CLI (`node scripts/cross-browser-baseline.js ` / `./scripts/cross-browser-baseline.sh `):** +- `capture [url] [--local] [--json] [--engines a,b]` — capture baselines for `visualBaselines.browsers` into `baselineDir` + record manifest. `capture.mode: "container"` (default) wraps itself in `docker run` (skipped when already in-container via `CBB_IN_CONTAINER=1`/`/.dockerenv`, or with `--local`, which records `host: "local"` + warns for firefox/webkit). +- `compare [url] [--json] [--current-dir ] [--blocking]` — backend `fetch()` → provenance verify → capture current (or use `--current-dir`, which makes the comparator fully testable without Playwright) → per-baseline `visual-diff.js --threshold ` → report to `.claude/visual-qa/cross-browser-report.md` + diffs to `diffDir`. Exit 0 unless (failures AND (`blocking` config or `--blocking`)); exit 2 on operational error. JSON: `{pass, fail, warn, skip, provenance: {...counts}, wouldBlock, blocking, threshold, backend, reportPath}`. +- `verify [--json]` — provenance-only check (no server needed). Exit 1 on any violation, 0 clean, 2 error. +- Missing baselines → skip + hint to run capture (never auto-capture; authoritative baselines are captured deliberately). + +**Backend adapter contract (`scripts/lib/baseline-backends.js`, RFC §8):** `resolveBackend(config, {execFile})` → `{ name, delegated, fetch({baselineDir}) → {baselineRoot}, store(ctx) → instructions/actions, providerArgv(ctx) }`. `commit`: fetch = configured dir; store = `git add` hint + LFS-drift warning. `ci-artifact`: fetch = download `cross-browser-baselines` artifact from last successful run of the baseline workflow on main (`gh run list`/`gh run download`; in CI, `actions/download-artifact`); store = artifact upload (CI). `service`: `delegated: true`; compare execs provider CLI (`chromatic --project-token ` / `percy snapshot `) after validating the token env var exists. + +**Playwright resolution (not a repo dep):** `createRequire` attempts against `$CBB_PLAYWRIGHT_DIR`, `process.cwd()`, script dir; clear install hint on failure. Container bootstrap installs `@playwright/test@` into a scratch dir and sets `CBB_PLAYWRIGHT_DIR` (image ships browsers at `/ms-playwright`). + +**Docker invocation (built in JS, unit-tested via injected spawn):** `docker run --rm -v :/work -w /work --add-host=host.docker.internal:host-gateway -e CBB_IN_CONTAINER=1 -e CBB_PLAYWRIGHT_DIR=/tmp/cbb bash -lc "mkdir -p /tmp/cbb && cd /tmp/cbb && npm init -y >/dev/null 2>&1 && npm i --no-fund --no-audit @playwright/test@ >/dev/null 2>&1 && cd /work && node scripts/cross-browser-baseline.js capture --host container --no-manifest"` with `localhost`/`127.0.0.1` in the URL rewritten to `host.docker.internal`. Manifest is recorded host-side afterwards (`host: "container"`, gitSha from host git). + +--- + +## Task 1: Accept RFC 0002 (gate-clearing) + +Issue #111 is blocked on RFC acceptance. PR #112 (the RFC) was approved and merged by the maintainer (PAMulligan) on 2026-07-01; per RFC §12 acceptance = maintainer approval + Status row update. Record it. + +**Files:** Modify `docs/rfcs/0002-cross-browser-screenshot-baseline-storage.md` (Status row, §12); Modify `docs/rfcs/README.md` (index row). + +1. RFC Status row → `| **Status** | **Accepted** — 2026-07-01 |`; §12: `- Decision: **Accepted as proposed**` / `- Approver / date: PAMulligan (maintainer) · 2026-07-01 (PR #112 approved & merged)`, and reword the §12 lead sentence + acceptance-gate blockquote (§ preamble) to past tense. +2. Index row in `docs/rfcs/README.md` → `Accepted`. +3. Commit: `docs(rfcs): accept RFC 0002 — cross-browser screenshot baseline storage` + +## Task 2: `visualBaselines` config + JSON Schema + validation tests (TDD) + +**Files:** Test `scripts/__tests__/visual-baselines-config.test.js` (new); Modify `.claude/pipeline.config.json` (after `regressionTesting`), `.claude/pipeline.config.schema.json` (property after `regressionTesting` + structural checks target), `scripts/validate-pipeline-config.js` (`structuralChecks()`). + +1. Write failing tests (black-box: `execFileSync("node", [VALIDATE_SCRIPT, "--config", tmpConfig, "--schema", SCHEMA, "--json"])` on temp configs derived from the live one): + - live config validates clean; + - unknown key inside `visualBaselines` → invalid (strictness); + - `backend: "s3"` → invalid enum; `storage: "lfs"` valid; `provenance.policy: "block"` → invalid enum (allowed: `warn`|`enforce`); + - structural: `backend: "ci-artifact"` + `storage: "lfs"` → error (storage applies to `commit` only); `visualBaselines.threshold: 0.05` with `e2e.crossBrowserDiffThreshold: 0.03` → error (single cross-engine tolerance, same style as the existing mutation-threshold drift check). +2. Run: `pnpm vitest run --config scripts/__tests__/vitest.config.js scripts/__tests__/visual-baselines-config.test.js` → FAIL. +3. Add the config section (values in **Contracts**/Design above: enabled, backend=commit, storage=git, baselineDir, screenshotDir/diffDir under `cross-browser`, browsers [chromium,firefox,webkit], routes ["/"], breakpoints {mobile:375,desktop:1440}, threshold 0.03, blocking false, reportFile, capture {mode:container, image:v1.61.1-noble, waitAfterLoadMs:1500, fullPage:true}, provenance {manifest, policy:warn}, ciArtifact {compareAgainst:last-green-main, retentionDays:30}, service {provider:chromatic, projectTokenEnv:CHROMATIC_PROJECT_TOKEN}); mirror in schema (strict, `$refs`: percentageRatio/browserList/breakpointMap; enums for backend/storage/mode/policy/provider/compareAgainst); implement the two structural checks. +4. Tests pass; also `node scripts/validate-pipeline-config.js` clean. Commit: `feat(config): add visualBaselines section for cross-browser baseline storage (RFC 0002)` + +## Task 3: Provenance manifest lib (TDD) + +**Files:** Test `scripts/__tests__/baseline-manifest.test.js`; Create `scripts/lib/baseline-manifest.js` (dual-mode: exports + CLI guard like `scripts/lib/pipeline-config.js`). + +Exports: `sha256File`, `parseBaselineRelPath` (`engine/routeSlug/bp_640px.png` → parts, else null), `loadManifest`, `recordBaselines({baselineDir, manifestPath, engines, envelope:{playwrightVersion,image,host,gitSha}, routesBySlug})` (scan configured engines' PNGs, merge entries — other engines' entries preserved; top-level envelope updated only when `envelope.updateToplevel !== false`), `syncManifest(...)` (refresh sha/capturedAt/gitSha/host for existing+new entries of given engines; used by chromium-writing scripts; no-op when manifest absent), `verifyBaselines({baselineDir, manifestPath, engines, envelope, image})` → `{statuses: {relPath: status}, counts, envelopeMatch}`. CLI: `node scripts/lib/baseline-manifest.js (record|sync|verify) [--json] [--engines a,b] [--host h]` reading `visualBaselines` config defaults. + +1. Failing tests: temp dir fixtures with tiny PNGs (reuse `scripts/__tests__/generate-fixtures.js` helpers) covering: record creates RFC-shaped manifest (path keys posix, engine/route/breakpoint parsed, sha256 correct); record preserves foreign-engine entries; sync no-ops without manifest; verify returns each status (`ok`/`untracked`/`modified`/`missing-file`/`foreign-host` for local firefox, `version-drift`, `image-drift`); envelopeMatch false when host differs; CLI `verify --json` exit codes (1 on violations). +2. Implement minimal; tests green. Commit: `feat(scripts): add baseline provenance manifest lib (record/sync/verify)` + +## Task 4: `cross-browser-baseline` CLI — compare/verify/capture (commit backend) (TDD) + +**Files:** Test `scripts/__tests__/cross-browser-baseline.test.js`; Create `scripts/cross-browser-baseline.js` (CLI; builtin-only static imports; Playwright dynamic; diff via `execFile node scripts/visual-diff.js`), `scripts/cross-browser-baseline.sh` (wrapper: common.sh, `visualBaselines.enabled` gate → `⊘ skip` exit 0, forwards to node CLI), `scripts/lib/baseline-backends.js` (commit backend + `resolveBackend` scaffold with clear errors for not-yet-implemented backends). Modify `.gitignore` (ignore `screenshots/cross-browser/**/*.png`, `diffs/cross-browser/**/*.png`, `cross-browser-report.md` under `.claude/visual-qa/`). + +1. Failing tests (all Playwright-free via `--current-dir` + `CBB_TEST_CONFIG` env pointing at a temp config file so tests don't depend on the live repo config): + - `--help` for sh + js; unknown backend in config → exit 2 with message; + - compare: identical current/baseline PNG trees → all PASS, exit 0, report file written, JSON shape as contracted; + - compare with a genuinely different PNG → fail counted, exit 0 while `blocking:false` + `wouldBlock:true`; exit 1 with `--blocking`; + - provenance: baseline modified after manifest record → `modified` flag surfaces in JSON + report; `policy:"enforce"` → excluded from diffing + counted as provenance failure; missing manifest → all `untracked` warnings (still diffs); + - engines filter: only `visualBaselines.browsers` subdirs are walked (a stray `chromium-extra/` dir is ignored); `manifest.json` itself skipped; + - missing baselines → skip + capture hint, exit 0; + - `verify` subcommand passes through to lib with config defaults. +2. Implement compare/verify + commit backend + wrapper; keep capture local-only in this task (`--local`/`mode:"local"`, chromium-capable; container wrapping is Task 6) — capture errors cleanly when Playwright unresolvable (install hint, exit 2). +3. Green; `bash -n scripts/cross-browser-baseline.sh`. Commit: `feat(scripts): add cross-browser-baseline capture/compare with provenance verification` + +## Task 5: Keep sibling writers honest + fix the mislabeled orchestration phase + +**Files:** Modify `scripts/regression-test.sh` (walk only `regressionTesting.browsers` subdirs; after `--update-baselines` copy, call manifest `sync --engines --host local` when a manifest exists), `scripts/capture-baselines.sh` (same sync after successful capture), `.claude/pipeline.config.json` (`orchestration.phases.cross-browser`: description → `"Firefox and WebKit baseline comparison (cross-browser-baseline.sh compare)"`, resources → `[{"name": "port:dev-server", "mode": "shared"}]`), `.claude/skills/parallel-orchestration/SKILL.md:148`, `.claude/skills/figma-to-react-workflow/SKILL.md:487-488,689`, `.claude/skills/visual-qa-verification/SKILL.md:148-155,352`, `.claude/commands/build-from-figma.md:285-286` (Phase 7 now runs the compare; `cross-browser-test.sh` remains a standalone capture utility — add a pointer comment to its header). + +1. Extend `scripts/__tests__/regression-test.test.js`: with a temp `CBB`-style fixture… (regression-test.sh reads the live config; simplest deterministic test: create temp baseline tree containing `firefox/` PNGs + chromium PNGs, run with a stubbed config via `PATH`-independent approach — if impractical black-box, assert via `bash -n` + a focused unit on the find filter by extracting it into `scripts/lib/common.sh` helper `common_find_baselines `; test the helper through a tiny bash -c harness). Keep whatever is honestly testable; don't fake it. +2. Apply edits; existing suite stays green (its `--help` tests unaffected). Commit: `fix(pipeline): make Phase 7 cross-browser phase perform the comparison it advertises` + +## Task 6: Pinned-container capture (Phase B) + +**Files:** Test additions in `scripts/__tests__/cross-browser-baseline.test.js`; Modify `scripts/cross-browser-baseline.js`. + +1. Failing tests for the docker command builder (exported or via `capture --dry-run --json` printing the argv): image from config; volume/workdir; `--add-host`; env plumbing; URL rewrite `http://localhost:3000` → `http://host.docker.internal:3000`; in-container detection short-circuits wrapping; `--local` bypass warns for firefox/webkit; image-tag/Playwright version drift warning surfaces. +2. Implement (`--dry-run` prints without spawning; real spawn inherits stdio, exit code propagated; post-success host-side manifest record with `host:"container"`). +3. Green. Commit: `feat(scripts): pinned Playwright container capture for cross-browser baselines` + +## Task 7: CI workflow + +**Files:** Create `.github/workflows/cross-browser-baselines.yml` (mirror `ci.yml` step conventions — read its visual-regression job first). + +Jobs: `prep` (bare runner; reads `visualBaselines` via `node scripts/lib/pipeline-config.js get …` → outputs `enabled`, `image`, `backend`, `pwversion`); `compare` (PR-only; needs prep; `container: image: ${{ needs.prep.outputs.image }}`; guards: baselines exist + `hashFiles('app/package.json') != ''` like the regression job; pnpm install (root), `npm i --prefix /tmp/cbb @playwright/test@` + `CBB_PLAYWRIGHT_DIR=/tmp/cbb`, build/start app, `node scripts/cross-browser-baseline.js compare http://localhost:3000 --json` (`continue-on-error` while non-blocking), upload report+diffs artifact, PR comment upsert titled "Cross-Browser Baseline Results"); `capture` (workflow_dispatch; same container; runs capture + opens a baselines-update PR via `gh` with `GITHUB_TOKEN`); `publish` (push→main; only when `backend == 'ci-artifact'`; capture + `actions/upload-artifact` name `cross-browser-baselines`, retention from config). + +Validate: `node -e "..."` YAML parse via `npx yaml` or python — use the same validation ci.yml gets (`validate` job does JSON only; locally run `npx --yes yaml-lint` or python yaml). Commit: `ci: add pinned-container cross-browser baseline workflow` + +## Task 8: Git LFS opt-in (Phase C.1) + +**Files:** Test `scripts/__tests__/setup-baseline-lfs.test.js`; Create `scripts/setup-baseline-lfs.sh`. + +1. Failing tests (run inside a scratch `git init` temp repo — never this repo; skipIf `git lfs version` unavailable for the apply-path test, keep pure-output tests always-on): `--help`; refuses when `storage` ≠ `lfs` without `--force`; `--dry-run` prints planned `.gitattributes` line without writing; apply path appends filter exactly once (idempotent on second run), runs `git lfs install --local`, prints forward-only vs `git lfs migrate import` guidance (never executes migrate). +2. Implement (`.gitattributes` line: `.claude/visual-qa/baselines/**/*.png filter=lfs diff=lfs merge=lfs -text`); also `cross-browser-baseline.js` capture/compare warn on storage↔attributes drift both directions (unit-covered in Task 4 file). Commit: `feat(scripts): git-lfs storage automation for large baseline sets` + +## Task 9: `ci-artifact` + `service` adapters (Phase C.2/C.3) + +**Files:** Test `scripts/__tests__/baseline-backends.test.js`; Modify `scripts/lib/baseline-backends.js`, `scripts/cross-browser-baseline.js` (wire `compare --backend` override for testing; JSON includes `backend`). + +1. Failing tests (inject fake `execFile` recorder): ci-artifact fetch builds `gh run list … --workflow cross-browser-baselines.yml --branch main --status success` then `gh run download -n cross-browser-baselines -D `; empty run list → clear "no published baselines yet" skip; service: missing token env → exit 2 with message naming the env var; chromatic/percy argv shapes; `delegated` short-circuits the pixel path. +2. Implement. Commit: `feat(scripts): ci-artifact and service baseline backends behind the adapter contract` + +## Task 10: Documentation + reference sweep + +**Files:** Create `docs/regression-testing/cross-browser.md` (guide: quick start, config table, provenance model + statuses, container capture local/CI, blocking-flip criteria, backend adapters incl. security/cost table from RFC §11, LFS adoption/migration, troubleshooting); Modify `docs/regression-testing/README.md` (cross-link + shared-baseline-dir note), `docs/onboarding/architecture.md:294-297` (script table rows), `docs/onboarding/pipeline-configuration.md` (visualBaselines key table), `scripts/README.md`, root `README.md` (scripts tree), `CLAUDE.md` (script list entries, Phase 7 line in the pipeline diagram, features bullet, architecture footer counts + Last Updated), this plan file committed, `docs/guides/error-recovery.md:31` if the phrasing about cross-browser capture needs the new script. + +Checks: `./scripts/check-doc-counts.sh` (agents/skills counts unchanged — must stay green), grep for stale claims (`"never diffs"` style wording) across touched docs. Commit: `docs: cross-browser baseline storage guide + reference sweep (closes #111 docs)` + +## Task 11: Full verification + PR + +1. `pnpm vitest run --config scripts/__tests__/vitest.config.js scripts/__tests__/` (known pre-existing failures on this machine: agent-plugin-lib/metrics-dashboard/pipeline-cache — verify unchanged vs `git stash` baseline if they appear). +2. `node scripts/validate-pipeline-config.js`; `bash -n` all new/modified shell scripts; `./scripts/lint-and-format.sh --check` (scope: touched files). +3. End-to-end smoke (local, honest): tiny static page served via `node -e` http server → `cross-browser-baseline.sh capture http://127.0.0.1: --local --engines chromium` (Playwright unavailable in-repo → expect the graceful install-hint path; if resolvable via npx-installed package, run for real) → fabricate the compare path via `--current-dir` fixtures instead, which exercises the full comparator+manifest+report pipeline without Playwright. +4. Push branch `111-implement-cross-browser-screenshot-baseline-storage-rfc-0002`, open PR: `feat: cross-browser screenshot baseline storage (RFC 0002)` — body maps commits → RFC phases A/B/C, lists §13 resolutions, notes what stays deliberately off (`blocking`, `crossBrowserScreenshotsRequired`), `Closes #111`. diff --git a/docs/regression-testing/README.md b/docs/regression-testing/README.md index ef41a77..09843a1 100644 --- a/docs/regression-testing/README.md +++ b/docs/regression-testing/README.md @@ -3,6 +3,12 @@ Automated screenshot comparison testing that captures pipeline output and compares against committed baseline screenshots to catch visual regressions. +> **Cross-browser baselines** (firefox/webkit, pinned-container capture, +> provenance manifest, pluggable storage backends) are the RFC 0002 flow +> documented in [cross-browser.md](cross-browser.md). This page covers the +> same-browser chromium regression flow; the two share the baseline directory +> but each walks only its own engines. + ## Quick Start ```bash @@ -69,15 +75,22 @@ The `visual-regression` job in `.github/workflows/ci.yml`: ``` .claude/visual-qa/ ├── baselines/ # Git-tracked baseline PNGs -│ └── chromium/ -│ └── home/ -│ ├── mobile_375px.png -│ └── desktop_1440px.png +│ ├── manifest.json # Cross-browser provenance (see cross-browser.md) +│ ├── chromium/ # This flow (regressionTesting.browsers) +│ │ └── home/ +│ │ ├── mobile_375px.png +│ │ └── desktop_1440px.png +│ ├── firefox/ # Cross-browser flow (visualBaselines) +│ └── webkit/ # Cross-browser flow (visualBaselines) ├── screenshots/regression/ # Transient current captures (gitignored) ├── diffs/regression/ # Transient diff images (gitignored) └── regression-report.md # Latest report ``` +When `--update-baselines` (or `capture-baselines.sh`) rewrites chromium +baselines and a cross-browser provenance manifest exists, the affected entries +are refreshed automatically so provenance stays truthful. + ## Thresholds - **Pass:** < 2% mismatch (configurable via `threshold`) diff --git a/docs/regression-testing/cross-browser.md b/docs/regression-testing/cross-browser.md new file mode 100644 index 0000000..f27b85f --- /dev/null +++ b/docs/regression-testing/cross-browser.md @@ -0,0 +1,176 @@ +# Cross-Browser Screenshot Baselines (RFC 0002) + +Firefox and WebKit render fonts, anti-aliasing, and form controls differently +from Chromium — and differently across operating systems. This flow stores +per-engine baselines, verifies **where each baseline came from** before +trusting it, and diffs current screenshots against them at a cross-engine +tolerance. It implements +[RFC 0002](../rfcs/0002-cross-browser-screenshot-baseline-storage.md); the +same-browser chromium flow stays in [regression testing](README.md). + +## Quick Start (commit backend, the default) + +```bash +# 1. Capture baselines deterministically in the pinned Playwright container +# (requires Docker; the dev server must be running) +./scripts/cross-browser-baseline.sh capture http://localhost:3000 + +# 2. Commit the PNGs together with their provenance manifest +git add .claude/visual-qa/baselines +git commit -m "test: add cross-browser baselines" + +# 3. Compare after changes (also runs as pipeline Phase 7 and in CI on PRs) +./scripts/cross-browser-baseline.sh compare http://localhost:3000 + +# Provenance-only check — no dev server needed +./scripts/cross-browser-baseline.sh verify +``` + +`capture --dry-run` prints the underlying `docker run` command without +executing it. `capture --local` skips the container: allowed, but the +baselines are recorded with `host: "local"` and flagged as *foreign* by every +subsequent compare — local firefox/webkit rendering is not reproducible in CI. + +## Why determinism comes first + +A stored baseline is only meaningful if the same input reproduces the same +pixels. Cross-engine, cross-OS rendering drift would otherwise turn every +compare into noise — regardless of where the PNGs are stored. Two mechanisms +make the signal trustworthy (RFC 0002 §5): + +1. **Pinned container capture.** Authoritative firefox/webkit baselines are + captured inside `visualBaselines.capture.image` + (`mcr.microsoft.com/playwright:v1.61.1-noble`), locally via `docker run` + and in CI via a `container:` job. The config pin is authoritative; scripts + warn when it drifts from the resolved `@playwright/test` version. +2. **Provenance manifest.** `baselines/manifest.json` (committed) records the + Playwright version, image, and per-baseline `sha256`, `capturedAt`, + `gitSha`, `engine`, `route`, `breakpoint`, and `host`. The comparator + verifies it before diffing. + +### Provenance statuses + +| Status | Meaning | +|--------|---------| +| `ok` | File matches its manifest entry and was captured in the container | +| `untracked` | Baseline PNG with no manifest entry (added outside capture) | +| `modified` | sha256 differs from the manifest (rewritten outside capture) | +| `missing-file` | Manifest entry whose PNG is gone | +| `foreign-host` | firefox/webkit baseline captured with `host: "local"` | + +Manifest-level checks: **version drift** (manifest Playwright ≠ current +envelope) and **image drift** (manifest image ≠ configured image). A compare +run from a non-container envelope is marked **advisory** — results are +informational, never authoritative. + +`provenance.policy` decides the teeth: `"warn"` (default) reports flags and +still pixel-diffs; `"enforce"` excludes flagged baselines from diffing and +counts them as failures instead of emitting false pixel diffs. + +## Configuration (`visualBaselines`) + +All settings live in `.claude/pipeline.config.json`; see the +[full key reference](../onboarding/pipeline-configuration.md#cross-browser-baselines-visualbaselines). +Highlights: + +| Setting | Default | Notes | +|---------|---------|-------| +| `backend` | `"commit"` | `commit` \| `ci-artifact` \| `service` | +| `storage` | `"git"` | commit backend only; `"lfs"` for large sets | +| `browsers` | `["chromium","firefox","webkit"]` | must stay within `e2e.crossBrowserBrowsers` | +| `threshold` | `0.03` | must equal `e2e.crossBrowserDiffThreshold` (validated) | +| `blocking` | `false` | flip only after determinism is proven (below) | +| `capture.mode` | `"container"` | `"local"` allowed but untrusted for firefox/webkit | +| `capture.image` | `mcr.microsoft.com/playwright:v1.61.1-noble` | the determinism pin | +| `provenance.policy` | `"warn"` | `"enforce"` excludes flagged baselines | + +## Storage backends (RFC 0002 §8) + +| | `commit` (default) | `ci-artifact` | `service` | +|---|---|---|---| +| Baselines live | in git, atomically with code | CI artifact from last green `main` | provider cloud | +| Review surface | PR diff of PNGs | artifact + PR comment | provider dashboard | +| External dependency | none | CI + `gh` CLI | paid SaaS + token | +| Data exposure | none | CI provider | screenshots sent to SaaS | + +- **commit** — the default: reviewable in the PR diff, offline-capable, zero + accounts. Pack growth is handled by the LFS flag (below). +- **ci-artifact** — no binaries in git. PR compares download the + `cross-browser-baselines` artifact published by the last green `main` run + (the `publish` job in `.github/workflows/cross-browser-baselines.yml`); + locating it needs the `gh` CLI. Before the first publish, compares skip + with a clear message. +- **service** — fully delegated to Chromatic or Percy: `compare` validates + the project token env var (`service.projectTokenEnv`, set it as a CI + secret) and hands off to the provider CLI. Chromatic mirrors + `blocking: false` via `--exit-zero-on-changes`; Percy snapshots the + configured routes. Opt-in only — screenshots leave your infrastructure. + +## Git LFS for large baseline sets + +Plain committed PNGs are right for the common case. Once a project's set +grows (rule of thumb: >50 baselines or ~25 MB cumulative), flip to LFS: + +```bash +# 1. Set visualBaselines.storage to "lfs" in pipeline.config.json +# 2. Run the automation in the app's repository +./scripts/setup-baseline-lfs.sh # idempotent; --dry-run to preview +``` + +The script appends the `.gitattributes` filter and runs +`git lfs install --local` — **forward-only**: existing history is untouched. +Rewriting history into LFS (`git lfs migrate import --include=... --everything`) +is destructive and intentionally left to a deliberate manual step; the script +prints the exact command. Capture/compare warn when `storage` and +`.gitattributes` disagree in either direction. + +## CI (`.github/workflows/cross-browser-baselines.yml`) + +- **PRs** — `compare` runs inside the pinned container (config-driven image), + uploads diff artifacts, and upserts a "Cross-Browser Baseline Results" PR + comment. Skips when no baselines or no `app/` exist. +- **workflow_dispatch** — deterministic recapture that opens a reviewable + baselines-refresh PR (PNGs + manifest). +- **push to main** — publishes the `cross-browser-baselines` artifact when + `backend` is `ci-artifact`. + +## Flipping to blocking (Phase B "teeth") + +`blocking` and `qualityGate.crossBrowserScreenshotsRequired` ship `false`. +Recommended flip criteria: **≥20 consecutive containerized CI compares with +zero provenance flags, and pixel failures only on intentional UI changes +(<5% of runs otherwise)**. Then set `visualBaselines.blocking: true` +(compare exits 1 on failures) and, once the pipeline gate should enforce it, +`qualityGate.crossBrowserScreenshotsRequired: true`. `provenance.policy: +"enforce"` is the matching escalation for provenance violations. + +## Relationship to same-browser regression testing + +Both flows share `.claude/visual-qa/baselines/` (per-engine subdirectories) +and `scripts/visual-diff.js`: + +| | Regression ([README](README.md)) | Cross-browser (this page) | +|---|---|---| +| Engines | `regressionTesting.browsers` (chromium) | `visualBaselines.browsers` (+ firefox/webkit) | +| Threshold | `0.02` | `0.03` (cross-engine tolerance) | +| Capture | local is fine (same-engine, same-host) | pinned container for authority | +| Provenance | syncs the manifest when present | records + verifies the manifest | + +Each script walks only its own engines, so committed firefox/webkit baselines +never appear as SKIP noise in the regression report. When +`regression-test.sh --update-baselines` or `capture-baselines.sh` rewrite +chromium baselines, they refresh the manifest entries (`host: "local"`) so the +provenance stays truthful. + +## Troubleshooting + +| Symptom | Cause / fix | +|---------|-------------| +| `compare` skips with "no baselines found" | Run `capture` (container) and commit `.claude/visual-qa/baselines/` | +| Everything flagged `untracked` | No manifest yet — run `capture`, or `node scripts/lib/baseline-manifest.js record` for existing PNGs | +| `foreign-host` on firefox/webkit | Baseline was captured with `--local`; recapture in the container | +| "advisory: current capture envelope does not match" | You're comparing locally against container baselines — expected; CI is authoritative | +| `@playwright/test is not resolvable` | Run from the app project (or install it): `pnpm add -D @playwright/test && npx playwright install` | +| `cannot derive a Playwright version from …capture.image` | Pin a full tag, e.g. `mcr.microsoft.com/playwright:v1.61.1-noble` | +| Container can't reach the dev server | localhost is rewritten to `host.docker.internal`; ensure the server listens on all interfaces | +| storage/LFS drift warning | Run `./scripts/setup-baseline-lfs.sh` (or set `storage` back to `git`) | diff --git a/docs/rfcs/0002-cross-browser-screenshot-baseline-storage.md b/docs/rfcs/0002-cross-browser-screenshot-baseline-storage.md index 4eb290a..d18089f 100644 --- a/docs/rfcs/0002-cross-browser-screenshot-baseline-storage.md +++ b/docs/rfcs/0002-cross-browser-screenshot-baseline-storage.md @@ -4,7 +4,7 @@ | ------------ | ----------------------------------------------------------------------- | | **RFC** | 0002 | | **Title** | Cross-Browser Screenshot Baseline Storage | -| **Status** | **Proposed** — awaiting review & acceptance | +| **Status** | **Accepted** — 2026-07-01 | | **Milestone**| v2.0.0 | | **Tracking** | #85 | | **Author** | Aurelius maintainers | @@ -13,9 +13,9 @@ > **Acceptance gate.** Per #85, the acceptance criteria are (1) the decision is > documented in `docs/rfcs/` and (2) implementation is tracked as a follow-up -> issue. This RFC satisfies (1). It is **Accepted** when a maintainer approves the -> PR and updates the Status row (see §12); the implementation lands in follow-up -> PRs that reference the tracking issue created for (2). +> issue. This RFC satisfies (1). It was **Accepted** on maintainer approval and +> merge of its PR (#112, 2026-07-01 — see §12); the implementation lands in +> follow-up PRs that reference the tracking issue created for (2), #111. --- @@ -356,12 +356,12 @@ accept the exposure and cost knowingly, via explicit config. ## 12. Acceptance & sign-off -This RFC is **Proposed**. It is **Accepted** when a maintainer approves the PR and -updates the Status row to `Accepted` with date and approver. On acceptance, the -follow-up implementation issue (§14) is the entry point for Phases A–C. +This RFC is **Accepted**. Acceptance = maintainer approval of the RFC PR plus this +Status-row update with date and approver. The follow-up implementation issue +(§14) is the entry point for Phases A–C. -- Decision: _pending_ -- Approver / date: _pending_ +- Decision: **Accepted as proposed** +- Approver / date: PAMulligan (maintainer) · 2026-07-01 — RFC PR #112 approved & merged ## 13. Open questions (for reviewers) diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md index 5b61e57..ef03a8d 100644 --- a/docs/rfcs/README.md +++ b/docs/rfcs/README.md @@ -25,4 +25,4 @@ features and fixes do not need one. | RFC | Title | Status | Milestone | | ---------------------------------------------------------------- | ----------------------------------------- | -------- | --------- | | [0001](0001-plugin-architecture.md) | Plugin Architecture for Custom Agents | Proposed | v2.0.0 | -| [0002](0002-cross-browser-screenshot-baseline-storage.md) | Cross-Browser Screenshot Baseline Storage | Proposed | v2.0.0 | +| [0002](0002-cross-browser-screenshot-baseline-storage.md) | Cross-Browser Screenshot Baseline Storage | Accepted | v2.0.0 | diff --git a/scripts/README.md b/scripts/README.md index 04a3d10..12d57f4 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -30,11 +30,22 @@ All scripts live in `scripts/` and are designed to run from the project root. - **Usage**: `./scripts/run-tests.sh` or `./scripts/run-tests.sh --watch` ### Cross-Browser Testing (`cross-browser-test.sh`) -- **Purpose**: Capture screenshots across browsers at standard breakpoints +- **Purpose**: Capture screenshots across browsers at standard breakpoints (ad-hoc; no comparison — see `cross-browser-baseline.sh` for the baseline-backed flow) - **Usage**: `./scripts/cross-browser-test.sh ` - **Browsers**: chromium, firefox, webkit - **Output**: Screenshots saved to `.claude/visual-qa/screenshots//` +### Cross-Browser Baselines (`cross-browser-baseline.sh`) +- **Purpose**: Capture firefox/webkit/chromium baselines in the pinned Playwright container and diff current screenshots against them with per-baseline provenance verification (RFC 0002) +- **Usage**: `./scripts/cross-browser-baseline.sh [url] [--json] [--engines a,b] [--local] [--current-dir dir] [--blocking] [--dry-run]` +- **Config**: `pipeline.config.json` → `visualBaselines` (backend, storage, threshold, pinned image, provenance policy) +- **Output**: Report at `.claude/visual-qa/cross-browser-report.md`; transient diffs under `.claude/visual-qa/diffs/cross-browser/` +- **Docs**: `docs/regression-testing/cross-browser.md` + +### Baseline Git LFS Setup (`setup-baseline-lfs.sh`) +- **Purpose**: Route the baseline directory through Git LFS for large baseline sets (`visualBaselines.storage: "lfs"`); idempotent, forward-only (history rewrite is printed as guidance, never executed) +- **Usage**: `./scripts/setup-baseline-lfs.sh [--dry-run] [--force]` (run inside the app's repository) + ### Setup Playwright (`setup-playwright.sh`) - **Purpose**: One-time setup for Playwright browser engines - **Usage**: `./scripts/setup-playwright.sh` diff --git a/scripts/__tests__/baseline-backends.test.js b/scripts/__tests__/baseline-backends.test.js new file mode 100644 index 0000000..717a6c5 --- /dev/null +++ b/scripts/__tests__/baseline-backends.test.js @@ -0,0 +1,129 @@ +// Tests for scripts/lib/baseline-backends.js — the RFC 0002 §8 backend +// adapter contract (commit | ci-artifact | service). External commands (gh, +// provider CLIs) are exercised through an injected exec recorder. +import { describe, it, expect } from "vitest"; +import { existsSync } from "fs"; +import { join } from "path"; + +import { resolveBackend, BackendError } from "../lib/baseline-backends.js"; + +const BASE_CONFIG = { + backend: "commit", + storage: "git", + baselineDir: ".claude/visual-qa/baselines", + routes: ["/", "/about"], + ciArtifact: { compareAgainst: "last-green-main", retentionDays: 30 }, + service: { provider: "chromatic", projectTokenEnv: "CBB_TEST_TOKEN" }, +}; + +function recorder(responses = {}) { + const calls = []; + const exec = (cmd, args) => { + calls.push([cmd, ...args]); + const key = `${cmd} ${args.slice(0, 2).join(" ")}`; + for (const [prefix, value] of Object.entries(responses)) { + if (key.startsWith(prefix)) { + if (value instanceof Error) throw value; + return value; + } + } + return ""; + }; + return { exec, calls }; +} + +describe("resolveBackend", () => { + it("returns the commit backend by default with the worktree as baseline root", async () => { + const backend = resolveBackend(BASE_CONFIG); + expect(backend.name).toBe("commit"); + expect(backend.delegated).toBe(false); + const fetched = await backend.fetch({ baselineDir: "/some/dir" }); + expect(fetched.baselineRoot).toBe("/some/dir"); + expect(fetched.manifestPath).toBeUndefined(); + }); + + it("throws BackendError for unknown backends", () => { + expect(() => resolveBackend({ ...BASE_CONFIG, backend: "s3" })).toThrow(BackendError); + }); +}); + +describe("ci-artifact backend", () => { + const config = { ...BASE_CONFIG, backend: "ci-artifact" }; + + it("downloads the last green main artifact via gh and points at it", async () => { + const { exec, calls } = recorder({ "gh run list": "12345\n" }); + const backend = resolveBackend(config, { execFile: exec }); + expect(backend.name).toBe("ci-artifact"); + + const fetched = await backend.fetch({ baselineDir: "/ignored" }); + expect(fetched.baselineRoot).toBeTruthy(); + expect(existsSync(fetched.baselineRoot)).toBe(true); + expect(fetched.manifestPath).toBe(join(fetched.baselineRoot, "manifest.json")); + + const list = calls.find((c) => c[1] === "run" && c[2] === "list"); + expect(list).toContain("cross-browser-baselines.yml"); + expect(list).toContain("--branch"); + expect(list).toContain("main"); + expect(list).toContain("--status"); + expect(list).toContain("success"); + + const download = calls.find((c) => c[1] === "run" && c[2] === "download"); + expect(download).toContain("12345"); + expect(download).toContain("cross-browser-baselines"); + }); + + it("signals a skip when no published artifact exists yet", async () => { + const { exec, calls } = recorder({ "gh run list": "\n" }); + const backend = resolveBackend(config, { execFile: exec }); + const fetched = await backend.fetch({ baselineDir: "/ignored" }); + expect(fetched.baselineRoot).toBeNull(); + expect(fetched.reason).toMatch(/no published/i); + expect(calls.some((c) => c[2] === "download")).toBe(false); + }); + + it("wraps a missing gh CLI in a BackendError", async () => { + const { exec } = recorder({ + "gh run list": Object.assign(new Error("ENOENT"), { code: "ENOENT" }), + }); + const backend = resolveBackend(config, { execFile: exec }); + await expect(backend.fetch({ baselineDir: "/ignored" })).rejects.toThrow(BackendError); + }); +}); + +describe("service backend", () => { + const config = { ...BASE_CONFIG, backend: "service" }; + + it("is delegated and validates the project token env var", () => { + const backend = resolveBackend(config, {}); + expect(backend.delegated).toBe(true); + expect(() => backend.requireToken({})).toThrow(/CBB_TEST_TOKEN/); + expect(() => backend.requireToken({ CBB_TEST_TOKEN: "tok" })).not.toThrow(); + }); + + it("builds the chromatic argv honoring blocking semantics", () => { + const backend = resolveBackend(config, {}); + const nonBlocking = backend.providerArgv({ blocking: false }); + expect(nonBlocking.slice(0, 3)).toEqual(["npx", "--yes", "chromatic"]); + expect(nonBlocking).toContain("--exit-zero-on-changes"); + const blocking = backend.providerArgv({ blocking: true }); + expect(blocking).not.toContain("--exit-zero-on-changes"); + }); + + it("builds the percy argv around a generated snapshots file", () => { + const percyConfig = { + ...config, + service: { provider: "percy", projectTokenEnv: "PERCY_TOKEN" }, + }; + const backend = resolveBackend(percyConfig, {}); + const argv = backend.providerArgv({ blocking: false, snapshotsFile: "/tmp/snap.json" }); + expect(argv).toContain("@percy/cli"); + expect(argv).toContain("snapshot"); + expect(argv).toContain("/tmp/snap.json"); + + const snapshots = JSON.parse(backend.buildSnapshots({ url: "http://localhost:3000" })); + expect(snapshots).toEqual([ + { name: "/", url: "http://localhost:3000/" }, + { name: "/about", url: "http://localhost:3000/about" }, + ]); + }); +}); diff --git a/scripts/__tests__/baseline-manifest.test.js b/scripts/__tests__/baseline-manifest.test.js new file mode 100644 index 0000000..a9aa3b1 --- /dev/null +++ b/scripts/__tests__/baseline-manifest.test.js @@ -0,0 +1,443 @@ +// Tests for scripts/lib/baseline-manifest.js — the RFC 0002 provenance +// manifest (record / sync / verify) for cross-browser screenshot baselines. +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { execFileSync } from "child_process"; +import { writeFileSync, readFileSync, mkdirSync, mkdtempSync, rmSync, existsSync } from "fs"; +import { createHash } from "crypto"; +import { join, dirname } from "path"; +import { tmpdir } from "os"; +import { fileURLToPath } from "url"; + +import { + sha256File, + parseBaselineRelPath, + recordBaselines, + syncManifest, + verifyBaselines, + loadManifest, +} from "../lib/baseline-manifest.js"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const LIB = join(__dirname, "..", "lib", "baseline-manifest.js"); + +let tmp; +let treeCount = 0; + +beforeAll(() => { + tmp = mkdtempSync(join(tmpdir(), "baseline-manifest-")); +}); + +afterAll(() => { + rmSync(tmp, { recursive: true, force: true }); +}); + +/** Create a baseline tree: { "firefox/home/desktop_1440px.png": "content", ... } */ +function makeTree(files) { + const root = join(tmp, `tree-${treeCount++}`); + const baselineDir = join(root, "baselines"); + for (const [rel, content] of Object.entries(files)) { + const p = join(baselineDir, ...rel.split("/")); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, content); + } + mkdirSync(baselineDir, { recursive: true }); + return { root, baselineDir, manifestPath: join(baselineDir, "manifest.json") }; +} + +const ENVELOPE = { + playwrightVersion: "1.61.1", + image: "mcr.microsoft.com/playwright:v1.61.1-noble", + host: "container", + gitSha: "abc1234", +}; + +describe("sha256File", () => { + it("hashes file bytes", () => { + const p = join(tmp, "hash-me.txt"); + writeFileSync(p, "hello"); + expect(sha256File(p)).toBe(createHash("sha256").update("hello").digest("hex")); + }); +}); + +describe("parseBaselineRelPath", () => { + it("parses engine/routeSlug/breakpoint_width", () => { + expect(parseBaselineRelPath("firefox/home/desktop_1440px.png")).toEqual({ + engine: "firefox", + routeSlug: "home", + breakpoint: "desktop", + width: 1440, + }); + }); + + it("handles breakpoint names containing underscores and windows separators", () => { + expect(parseBaselineRelPath("webkit\\about-us\\small_mobile_320px.png")).toEqual({ + engine: "webkit", + routeSlug: "about-us", + breakpoint: "small_mobile", + width: 320, + }); + }); + + it("returns null for non-baseline paths", () => { + expect(parseBaselineRelPath("manifest.json")).toBeNull(); + expect(parseBaselineRelPath("firefox/loose.png")).toBeNull(); + expect(parseBaselineRelPath("firefox/home/desktop.png")).toBeNull(); + }); +}); + +describe("recordBaselines", () => { + it("writes an RFC-shaped manifest for the requested engines only", () => { + const { baselineDir, manifestPath } = makeTree({ + "chromium/home/mobile_375px.png": "c1", + "firefox/home/desktop_1440px.png": "f1", + "webkit/home/mobile_375px.png": "w1", + }); + const manifest = recordBaselines({ + baselineDir, + manifestPath, + engines: ["firefox", "webkit"], + envelope: ENVELOPE, + routes: ["/"], + }); + + expect(existsSync(manifestPath)).toBe(true); + expect(manifest.playwrightVersion).toBe("1.61.1"); + expect(manifest.image).toBe(ENVELOPE.image); + expect(Object.keys(manifest.baselines).sort()).toEqual([ + "firefox/home/desktop_1440px.png", + "webkit/home/mobile_375px.png", + ]); + + const entry = manifest.baselines["firefox/home/desktop_1440px.png"]; + expect(entry.sha256).toBe(createHash("sha256").update("f1").digest("hex")); + expect(entry.engine).toBe("firefox"); + expect(entry.route).toBe("/"); + expect(entry.breakpoint).toBe("desktop"); + expect(entry.host).toBe("container"); + expect(entry.gitSha).toBe("abc1234"); + expect(new Date(entry.capturedAt).toISOString()).toBe(entry.capturedAt); + }); + + it("maps route slugs from the provided routes and nulls unknown slugs", () => { + const { baselineDir, manifestPath } = makeTree({ + "firefox/about/mobile_375px.png": "a", + "firefox/mystery/mobile_375px.png": "m", + }); + const manifest = recordBaselines({ + baselineDir, + manifestPath, + engines: ["firefox"], + envelope: ENVELOPE, + routes: ["/", "/about"], + }); + expect(manifest.baselines["firefox/about/mobile_375px.png"].route).toBe("/about"); + expect(manifest.baselines["firefox/mystery/mobile_375px.png"].route).toBeNull(); + }); + + it("preserves other engines' entries and drops stale ones for scanned engines", () => { + const { baselineDir, manifestPath } = makeTree({ + "firefox/home/desktop_1440px.png": "f1", + "firefox/home/mobile_375px.png": "f2", + "webkit/home/mobile_375px.png": "w1", + }); + recordBaselines({ + baselineDir, + manifestPath, + engines: ["firefox", "webkit"], + envelope: ENVELOPE, + routes: ["/"], + }); + + // A firefox baseline disappears; re-record firefox only. + rmSync(join(baselineDir, "firefox", "home", "mobile_375px.png")); + const manifest = recordBaselines({ + baselineDir, + manifestPath, + engines: ["firefox"], + envelope: { ...ENVELOPE, gitSha: "def5678" }, + routes: ["/"], + }); + + expect(Object.keys(manifest.baselines).sort()).toEqual([ + "firefox/home/desktop_1440px.png", + "webkit/home/mobile_375px.png", + ]); + // webkit untouched, firefox refreshed + expect(manifest.baselines["webkit/home/mobile_375px.png"].gitSha).toBe("abc1234"); + expect(manifest.baselines["firefox/home/desktop_1440px.png"].gitSha).toBe("def5678"); + }); +}); + +describe("syncManifest", () => { + it("is a no-op when no manifest exists", () => { + const { baselineDir, manifestPath } = makeTree({ + "chromium/home/mobile_375px.png": "c1", + }); + const result = syncManifest({ + baselineDir, + manifestPath, + engines: ["chromium"], + host: "local", + gitSha: "abc1234", + }); + expect(result).toBeNull(); + expect(existsSync(manifestPath)).toBe(false); + }); + + it("refreshes entries for its engines without touching the envelope or other engines", () => { + const { baselineDir, manifestPath } = makeTree({ + "chromium/home/mobile_375px.png": "c1", + "firefox/home/mobile_375px.png": "f1", + }); + recordBaselines({ + baselineDir, + manifestPath, + engines: ["chromium", "firefox"], + envelope: ENVELOPE, + routes: ["/"], + }); + + // regression-test.sh rewrites the chromium baseline locally... + writeFileSync(join(baselineDir, "chromium", "home", "mobile_375px.png"), "c2"); + // ...and a new chromium baseline appears. + mkdirSync(join(baselineDir, "chromium", "about"), { recursive: true }); + writeFileSync(join(baselineDir, "chromium", "about", "mobile_375px.png"), "c3"); + + const manifest = syncManifest({ + baselineDir, + manifestPath, + engines: ["chromium"], + host: "local", + gitSha: "def5678", + routes: ["/", "/about"], + }); + + const updated = manifest.baselines["chromium/home/mobile_375px.png"]; + expect(updated.sha256).toBe(createHash("sha256").update("c2").digest("hex")); + expect(updated.host).toBe("local"); + expect(updated.gitSha).toBe("def5678"); + expect(manifest.baselines["chromium/about/mobile_375px.png"]).toBeDefined(); + // firefox entry and top-level envelope untouched + expect(manifest.baselines["firefox/home/mobile_375px.png"].host).toBe("container"); + expect(manifest.playwrightVersion).toBe("1.61.1"); + expect(manifest.image).toBe(ENVELOPE.image); + }); +}); + +describe("verifyBaselines", () => { + function recordedTree() { + const tree = makeTree({ + "firefox/home/desktop_1440px.png": "f1", + "webkit/home/mobile_375px.png": "w1", + }); + recordBaselines({ + baselineDir: tree.baselineDir, + manifestPath: tree.manifestPath, + engines: ["firefox", "webkit"], + envelope: ENVELOPE, + routes: ["/"], + }); + return tree; + } + + const CURRENT = { playwrightVersion: "1.61.1", host: "container" }; + const IMAGE = ENVELOPE.image; + + it("reports ok for pristine container-captured baselines", () => { + const { baselineDir, manifestPath } = recordedTree(); + const result = verifyBaselines({ + baselineDir, + manifestPath, + engines: ["firefox", "webkit"], + envelope: CURRENT, + image: IMAGE, + }); + expect(result.manifestFound).toBe(true); + expect(result.statuses["firefox/home/desktop_1440px.png"]).toBe("ok"); + expect(result.statuses["webkit/home/mobile_375px.png"]).toBe("ok"); + expect(result.violations).toBe(0); + expect(result.envelopeMatch).toBe(true); + expect(result.drift).toEqual({ version: false, image: false }); + }); + + it("flags modified, untracked, and missing baselines", () => { + const { baselineDir, manifestPath } = recordedTree(); + writeFileSync(join(baselineDir, "firefox", "home", "desktop_1440px.png"), "f1-EDITED"); + mkdirSync(join(baselineDir, "webkit", "about"), { recursive: true }); + writeFileSync(join(baselineDir, "webkit", "about", "mobile_375px.png"), "new"); + rmSync(join(baselineDir, "webkit", "home", "mobile_375px.png")); + + const result = verifyBaselines({ + baselineDir, + manifestPath, + engines: ["firefox", "webkit"], + envelope: CURRENT, + image: IMAGE, + }); + expect(result.statuses["firefox/home/desktop_1440px.png"]).toBe("modified"); + expect(result.statuses["webkit/about/mobile_375px.png"]).toBe("untracked"); + expect(result.statuses["webkit/home/mobile_375px.png"]).toBe("missing-file"); + expect(result.violations).toBe(3); + }); + + it("flags local-captured firefox/webkit baselines as foreign-host but not chromium", () => { + const { baselineDir, manifestPath } = makeTree({ + "chromium/home/mobile_375px.png": "c1", + "firefox/home/mobile_375px.png": "f1", + }); + recordBaselines({ + baselineDir, + manifestPath, + engines: ["chromium", "firefox"], + envelope: { ...ENVELOPE, host: "local" }, + routes: ["/"], + }); + const result = verifyBaselines({ + baselineDir, + manifestPath, + engines: ["chromium", "firefox"], + envelope: CURRENT, + image: IMAGE, + }); + expect(result.statuses["firefox/home/mobile_375px.png"]).toBe("foreign-host"); + expect(result.statuses["chromium/home/mobile_375px.png"]).toBe("ok"); + expect(result.violations).toBe(1); + }); + + it("reports version and image drift against the manifest envelope", () => { + const { baselineDir, manifestPath } = recordedTree(); + const result = verifyBaselines({ + baselineDir, + manifestPath, + engines: ["firefox", "webkit"], + envelope: { playwrightVersion: "1.62.0", host: "container" }, + image: "mcr.microsoft.com/playwright:v1.62.0-noble", + }); + expect(result.drift).toEqual({ version: true, image: true }); + expect(result.envelopeMatch).toBe(false); + expect(result.violations).toBeGreaterThanOrEqual(2); + }); + + it("marks a local current envelope as not matching (advisory compares)", () => { + const { baselineDir, manifestPath } = recordedTree(); + const result = verifyBaselines({ + baselineDir, + manifestPath, + engines: ["firefox", "webkit"], + envelope: { playwrightVersion: "1.61.1", host: "local" }, + image: IMAGE, + }); + expect(result.envelopeMatch).toBe(false); + // host mismatch alone is advisory, not a per-baseline violation + expect(result.statuses["firefox/home/desktop_1440px.png"]).toBe("ok"); + }); + + it("treats every baseline as untracked when no manifest exists", () => { + const { baselineDir, manifestPath } = makeTree({ + "firefox/home/desktop_1440px.png": "f1", + }); + const result = verifyBaselines({ + baselineDir, + manifestPath, + engines: ["firefox"], + envelope: CURRENT, + image: IMAGE, + }); + expect(result.manifestFound).toBe(false); + expect(result.statuses["firefox/home/desktop_1440px.png"]).toBe("untracked"); + expect(result.violations).toBe(1); + }); + + it("is clean when there is nothing to verify", () => { + const { baselineDir, manifestPath } = makeTree({}); + const result = verifyBaselines({ + baselineDir, + manifestPath, + engines: ["firefox", "webkit"], + envelope: CURRENT, + image: IMAGE, + }); + expect(result.manifestFound).toBe(false); + expect(result.violations).toBe(0); + }); +}); + +describe("CLI", () => { + function runCli(args, options = {}) { + try { + const stdout = execFileSync("node", [LIB, ...args], { + encoding: "utf-8", + timeout: 30000, + ...options, + }); + return { stdout, exitCode: 0 }; + } catch (err) { + return { stdout: err.stdout ?? "", exitCode: err.status }; + } + } + + it("verify --json exits 0 on a clean recorded tree", () => { + const { baselineDir, manifestPath } = makeTree({ + "firefox/home/desktop_1440px.png": "f1", + }); + recordBaselines({ + baselineDir, + manifestPath, + engines: ["firefox"], + envelope: ENVELOPE, + routes: ["/"], + }); + const { stdout, exitCode } = runCli([ + "verify", + "--json", + "--baseline-dir", + baselineDir, + "--manifest", + manifestPath, + "--engines", + "firefox", + "--host", + "container", + "--playwright-version", + "1.61.1", + "--image", + ENVELOPE.image, + ]); + expect(exitCode).toBe(0); + const parsed = JSON.parse(stdout); + expect(parsed.violations).toBe(0); + }); + + it("verify --json exits 1 when a baseline was modified outside capture", () => { + const { baselineDir, manifestPath } = makeTree({ + "firefox/home/desktop_1440px.png": "f1", + }); + recordBaselines({ + baselineDir, + manifestPath, + engines: ["firefox"], + envelope: ENVELOPE, + routes: ["/"], + }); + writeFileSync(join(baselineDir, "firefox", "home", "desktop_1440px.png"), "tampered"); + const { stdout, exitCode } = runCli([ + "verify", + "--json", + "--baseline-dir", + baselineDir, + "--manifest", + manifestPath, + "--engines", + "firefox", + "--host", + "container", + "--playwright-version", + "1.61.1", + "--image", + ENVELOPE.image, + ]); + expect(exitCode).toBe(1); + const parsed = JSON.parse(stdout); + expect(parsed.statuses["firefox/home/desktop_1440px.png"]).toBe("modified"); + }); +}); diff --git a/scripts/__tests__/cross-browser-baseline.test.js b/scripts/__tests__/cross-browser-baseline.test.js new file mode 100644 index 0000000..902bbed --- /dev/null +++ b/scripts/__tests__/cross-browser-baseline.test.js @@ -0,0 +1,516 @@ +// Tests for scripts/cross-browser-baseline.js — RFC 0002 cross-browser +// baseline capture/compare/verify CLI (commit backend). Black-box via CLI; +// Playwright-free thanks to compare --current-dir. Config is injected per +// test through the CBB_CONFIG env var. +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { spawnSync } from "child_process"; +import { writeFileSync, existsSync, mkdirSync, mkdtempSync, rmSync, readFileSync } from "fs"; +import { join, dirname } from "path"; +import { tmpdir } from "os"; +import { fileURLToPath } from "url"; +import { PNG } from "pngjs"; +import { createRequire } from "module"; + +import { createPNG, solid } from "./generate-fixtures.js"; +import { recordBaselines } from "../lib/baseline-manifest.js"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const JS_CLI = join(__dirname, "..", "cross-browser-baseline.js"); +const SH_CLI = join(__dirname, "..", "cross-browser-baseline.sh"); +const IMAGE = "mcr.microsoft.com/playwright:v1.61.1-noble"; + +let tmp; +let projCount = 0; + +beforeAll(() => { + tmp = mkdtempSync(join(tmpdir(), "cbb-")); +}); + +afterAll(() => { + rmSync(tmp, { recursive: true, force: true }); +}); + +function pngBytes(rgb) { + return PNG.sync.write(createPNG(40, 30, solid(...rgb))); +} + +const RED = [200, 40, 40]; +const BLUE = [40, 40, 200]; + +/** Lay out a project: baselines + currents trees and a CBB_CONFIG file. */ +function makeProject({ config = {}, baselines = {}, currents = {} } = {}) { + const root = join(tmp, `proj-${projCount++}`); + const baselineDir = join(root, "baselines"); + const currentDir = join(root, "current"); + const paths = { + root, + baselineDir, + currentDir, + screenshotDir: join(root, "shots"), + diffDir: join(root, "diffs"), + reportFile: join(root, "cross-browser-report.md"), + manifestPath: join(baselineDir, "manifest.json"), + }; + const writeTree = (dir, files) => { + mkdirSync(dir, { recursive: true }); + for (const [rel, bytes] of Object.entries(files)) { + const p = join(dir, ...rel.split("/")); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, bytes); + } + }; + writeTree(baselineDir, baselines); + writeTree(currentDir, currents); + + const visualBaselines = { + enabled: true, + backend: "commit", + storage: "git", + baselineDir, + screenshotDir: paths.screenshotDir, + diffDir: paths.diffDir, + browsers: ["chromium", "firefox", "webkit"], + routes: ["/"], + breakpoints: { mobile: 375, desktop: 1440 }, + threshold: 0.03, + blocking: false, + reportFile: paths.reportFile, + capture: { mode: "local", image: IMAGE, waitAfterLoadMs: 0, fullPage: true }, + provenance: { manifest: paths.manifestPath, policy: "warn" }, + ciArtifact: { compareAgainst: "last-green-main", retentionDays: 30 }, + service: { provider: "chromatic", projectTokenEnv: "CHROMATIC_PROJECT_TOKEN" }, + ...config, + }; + paths.configPath = join(root, "config.json"); + writeFileSync(paths.configPath, JSON.stringify({ visualBaselines }, null, 2)); + return paths; +} + +function record(proj, overrides = {}) { + recordBaselines({ + baselineDir: proj.baselineDir, + manifestPath: proj.manifestPath, + engines: ["chromium", "firefox", "webkit"], + envelope: { + playwrightVersion: "1.61.1", + image: IMAGE, + host: "container", + gitSha: "testsha", + ...overrides, + }, + routes: ["/"], + }); +} + +function run(cliArgs, proj, { viaShell = false, env = {} } = {}) { + const [bin, prefix] = viaShell ? ["bash", [SH_CLI]] : ["node", [JS_CLI]]; + const result = spawnSync(bin, [...prefix, ...cliArgs], { + encoding: "utf-8", + timeout: 30000, + env: { ...process.env, ...(proj ? { CBB_CONFIG: proj.configPath } : {}), ...env }, + }); + return { stdout: result.stdout ?? "", stderr: result.stderr ?? "", exitCode: result.status }; +} + +function runJson(cliArgs, proj, options = {}) { + const result = run(cliArgs, proj, options); + return { ...result, json: result.stdout ? JSON.parse(result.stdout) : null }; +} + +const playwrightResolvable = (() => { + try { + createRequire(join(process.cwd(), "package.json")).resolve("@playwright/test"); + return true; + } catch { + return false; + } +})(); + +describe("help and usage", () => { + it("node CLI prints usage on --help", () => { + const { stdout, exitCode } = run(["--help"]); + expect(exitCode).toBe(0); + expect(stdout).toMatch(/capture/); + expect(stdout).toMatch(/compare/); + expect(stdout).toMatch(/verify/); + }); + + it("shell wrapper forwards --help", () => { + const { stdout, exitCode } = run(["--help"], null, { viaShell: true }); + expect(exitCode).toBe(0); + expect(stdout).toMatch(/compare/); + }); + + it("exits 2 with usage when no subcommand is given", () => { + const { exitCode, stderr } = run([]); + expect(exitCode).toBe(2); + expect(stderr).toMatch(/Usage/i); + }); +}); + +describe("compare (commit backend)", () => { + it("passes when current screenshots match committed baselines", () => { + const proj = makeProject({ + baselines: { + "firefox/home/desktop_1440px.png": pngBytes(RED), + "webkit/home/mobile_375px.png": pngBytes(BLUE), + }, + currents: { + "firefox/home/desktop_1440px.png": pngBytes(RED), + "webkit/home/mobile_375px.png": pngBytes(BLUE), + }, + }); + record(proj); + const { json, exitCode } = runJson( + ["compare", "--json", "--current-dir", proj.currentDir, "--host", "container"], + proj, + ); + expect(exitCode).toBe(0); + expect(json.backend).toBe("commit"); + expect(json.pass).toBe(2); + expect(json.fail).toBe(0); + expect(json.wouldBlock).toBe(false); + expect(json.provenance.violations).toBe(0); + expect(json.advisory).toBe(false); + expect(existsSync(proj.reportFile)).toBe(true); + expect(readFileSync(proj.reportFile, "utf-8")).toMatch(/Cross-Browser/i); + }); + + it("counts pixel failures but stays exit 0 while non-blocking", () => { + const proj = makeProject({ + baselines: { "firefox/home/desktop_1440px.png": pngBytes(RED) }, + currents: { "firefox/home/desktop_1440px.png": pngBytes(BLUE) }, + }); + record(proj); + const { json, exitCode } = runJson( + ["compare", "--json", "--current-dir", proj.currentDir, "--host", "container"], + proj, + ); + expect(exitCode).toBe(0); + expect(json.fail).toBe(1); + expect(json.wouldBlock).toBe(true); + expect(json.blocking).toBe(false); + }); + + it("exits 1 on failures with --blocking", () => { + const proj = makeProject({ + baselines: { "firefox/home/desktop_1440px.png": pngBytes(RED) }, + currents: { "firefox/home/desktop_1440px.png": pngBytes(BLUE) }, + }); + record(proj); + const { exitCode, json, stderr } = runJson( + ["compare", "--json", "--blocking", "--current-dir", proj.currentDir, "--host", "container"], + proj, + ); + expect(exitCode).toBe(1); + expect(json.blocking).toBe(true); + // visual-diff.js emits a mismatch RATIO; the CLI must report a true percent + // (a fully-different image is 100%, not "1%"). + expect(json.results[0].mismatchRatio).toBe(1); + expect(stderr).toContain("(100.00%)"); + expect(readFileSync(proj.reportFile, "utf-8")).toContain("100.00%"); + }); + + it("exits 1 on failures when config sets blocking: true (Phase B teeth)", () => { + const proj = makeProject({ + config: { blocking: true }, + baselines: { "webkit/home/mobile_375px.png": pngBytes(RED) }, + currents: { "webkit/home/mobile_375px.png": pngBytes(BLUE) }, + }); + record(proj); + const { exitCode } = runJson( + ["compare", "--json", "--current-dir", proj.currentDir, "--host", "container"], + proj, + ); + expect(exitCode).toBe(1); + }); + + it("reports modified provenance as a warning (policy=warn) while still diffing", () => { + const tampered = pngBytes(RED); + const proj = makeProject({ + baselines: { "firefox/home/desktop_1440px.png": pngBytes(BLUE) }, + currents: { "firefox/home/desktop_1440px.png": tampered }, + }); + record(proj); + // Rewrite the baseline after recording provenance → sha mismatch. + writeFileSync(join(proj.baselineDir, "firefox", "home", "desktop_1440px.png"), tampered); + const { json, exitCode } = runJson( + ["compare", "--json", "--current-dir", proj.currentDir, "--host", "container"], + proj, + ); + expect(exitCode).toBe(0); + expect(json.provenance.violations).toBeGreaterThanOrEqual(1); + const entry = json.results.find((r) => r.path === "firefox/home/desktop_1440px.png"); + expect(entry.provenance).toBe("modified"); + expect(entry.status).toBe("PASS"); // still pixel-diffed under warn policy + expect(json.provenanceFailures).toBe(0); + }); + + it("excludes flagged baselines from diffing under policy=enforce", () => { + const tampered = pngBytes(RED); + const proj = makeProject({ + config: { provenance: { manifest: undefined, policy: "enforce" } }, + baselines: { "firefox/home/desktop_1440px.png": pngBytes(BLUE) }, + currents: { "firefox/home/desktop_1440px.png": tampered }, + }); + // provenance.manifest was clobbered by the override — restore the path + const configRaw = JSON.parse(readFileSync(proj.configPath, "utf-8")); + configRaw.visualBaselines.provenance = { manifest: proj.manifestPath, policy: "enforce" }; + writeFileSync(proj.configPath, JSON.stringify(configRaw, null, 2)); + + record(proj); + writeFileSync(join(proj.baselineDir, "firefox", "home", "desktop_1440px.png"), tampered); + const { json, exitCode } = runJson( + ["compare", "--json", "--current-dir", proj.currentDir, "--host", "container"], + proj, + ); + expect(exitCode).toBe(0); // blocking still false + const entry = json.results.find((r) => r.path === "firefox/home/desktop_1440px.png"); + expect(entry.status).toBe("PROVENANCE"); + expect(json.provenanceFailures).toBe(1); + expect(json.wouldBlock).toBe(true); + }); + + it("marks compares advisory when the manifest is missing (all untracked)", () => { + const proj = makeProject({ + baselines: { "firefox/home/desktop_1440px.png": pngBytes(RED) }, + currents: { "firefox/home/desktop_1440px.png": pngBytes(RED) }, + }); + const { json, exitCode } = runJson( + ["compare", "--json", "--current-dir", proj.currentDir, "--host", "container"], + proj, + ); + expect(exitCode).toBe(0); + expect(json.provenance.manifestFound).toBe(false); + expect(json.advisory).toBe(true); + expect(json.pass).toBe(1); + }); + + it("marks local-envelope compares advisory", () => { + const proj = makeProject({ + baselines: { "firefox/home/desktop_1440px.png": pngBytes(RED) }, + currents: { "firefox/home/desktop_1440px.png": pngBytes(RED) }, + }); + record(proj); + const { json } = runJson( + ["compare", "--json", "--current-dir", proj.currentDir, "--host", "local"], + proj, + ); + expect(json.advisory).toBe(true); + expect(json.pass).toBe(1); + }); + + it("walks only configured engines and skips stray directories", () => { + const proj = makeProject({ + baselines: { + "firefox/home/desktop_1440px.png": pngBytes(RED), + "not-a-browser/home/desktop_1440px.png": pngBytes(RED), + }, + currents: { "firefox/home/desktop_1440px.png": pngBytes(RED) }, + }); + record(proj); + const { json } = runJson( + ["compare", "--json", "--current-dir", proj.currentDir, "--host", "container"], + proj, + ); + expect(json.results.map((r) => r.path)).toEqual(["firefox/home/desktop_1440px.png"]); + }); + + it("honors --engines filtering", () => { + const proj = makeProject({ + baselines: { + "firefox/home/desktop_1440px.png": pngBytes(RED), + "webkit/home/mobile_375px.png": pngBytes(BLUE), + }, + currents: { "firefox/home/desktop_1440px.png": pngBytes(RED) }, + }); + record(proj); + const { json } = runJson( + [ + "compare", + "--json", + "--engines", + "firefox", + "--current-dir", + proj.currentDir, + "--host", + "container", + ], + proj, + ); + expect(json.results).toHaveLength(1); + expect(json.results[0].engine).toBe("firefox"); + }); + + it("skips baselines with no current screenshot", () => { + const proj = makeProject({ + baselines: { + "firefox/home/desktop_1440px.png": pngBytes(RED), + "firefox/home/mobile_375px.png": pngBytes(RED), + }, + currents: { "firefox/home/desktop_1440px.png": pngBytes(RED) }, + }); + record(proj); + const { json, exitCode } = runJson( + ["compare", "--json", "--current-dir", proj.currentDir, "--host", "container"], + proj, + ); + expect(exitCode).toBe(0); + expect(json.skip).toBe(1); + expect(json.pass).toBe(1); + }); + + it("exits 0 with a capture hint when no baselines exist", () => { + const proj = makeProject({ currents: {} }); + const { json, exitCode } = runJson( + ["compare", "--json", "--current-dir", proj.currentDir], + proj, + ); + expect(exitCode).toBe(0); + expect(json.skipped).toBe(true); + expect(json.reason).toMatch(/no baselines/i); + }); + + it("exits 0 and reports skipped when visualBaselines.enabled is false", () => { + const proj = makeProject({ config: { enabled: false } }); + const { json, exitCode } = runJson(["compare", "--json"], proj); + expect(exitCode).toBe(0); + expect(json.skipped).toBe(true); + expect(json.reason).toMatch(/disabled/i); + }); + + it("warns when storage=lfs but no LFS filter covers the baselines", () => { + // The framework repo's .gitattributes has no LFS rule, so storage=lfs + // must surface a drift warning (setup-baseline-lfs.sh not run yet). + const proj = makeProject({ config: { storage: "lfs" } }); + const { stderr, exitCode } = run(["compare", "--json"], proj); + expect(exitCode).toBe(0); + expect(stderr).toMatch(/lfs/i); + expect(stderr).toMatch(/setup-baseline-lfs/); + }); + + it("exits 2 for an unknown backend", () => { + const proj = makeProject({ config: { backend: "carrier-pigeon" } }); + const { exitCode, stderr } = run(["compare", "--json"], proj); + expect(exitCode).toBe(2); + expect(stderr).toMatch(/backend/i); + }); + + it("service backend exits 2 with the env var name when the token is missing", () => { + const proj = makeProject({ + config: { + backend: "service", + service: { provider: "chromatic", projectTokenEnv: "CBB_MISSING_TOKEN_XYZ" }, + }, + }); + const { exitCode, stderr } = run(["compare", "--json"], proj); + expect(exitCode).toBe(2); + expect(stderr).toContain("CBB_MISSING_TOKEN_XYZ"); + }); +}); + +describe("verify", () => { + it("exits 0 on a clean recorded tree and 1 after tampering", () => { + const proj = makeProject({ + baselines: { "firefox/home/desktop_1440px.png": pngBytes(RED) }, + }); + record(proj); + expect(run(["verify", "--json", "--host", "container"], proj).exitCode).toBe(0); + + writeFileSync(join(proj.baselineDir, "firefox", "home", "desktop_1440px.png"), pngBytes(BLUE)); + const { exitCode, json } = runJson(["verify", "--json", "--host", "container"], proj); + expect(exitCode).toBe(1); + expect(json.statuses["firefox/home/desktop_1440px.png"]).toBe("modified"); + }); +}); + +describe("capture", () => { + it.skipIf(playwrightResolvable)( + "fails with an install hint when Playwright is not resolvable", + () => { + const proj = makeProject({}); + const { exitCode, stderr } = run(["capture", "http://127.0.0.1:9", "--local"], proj); + expect(exitCode).toBe(2); + expect(stderr).toMatch(/@playwright\/test/); + }, + ); +}); + +describe("capture — pinned container wrapping (Phase B)", () => { + const containerConfig = { + capture: { mode: "container", image: IMAGE, waitAfterLoadMs: 0, fullPage: true }, + }; + + it("builds the docker run command with the pinned image and rewritten URL (--dry-run)", () => { + const proj = makeProject({ config: containerConfig }); + const { json, exitCode } = runJson( + ["capture", "http://localhost:3000", "--dry-run", "--json"], + proj, + ); + expect(exitCode).toBe(0); + expect(json.dryRun).toBe(true); + expect(json.image).toBe(IMAGE); + expect(json.playwrightVersion).toBe("1.61.1"); + expect(json.url).toBe("http://host.docker.internal:3000"); + + const argv = json.docker; + expect(argv.slice(0, 3)).toEqual(["docker", "run", "--rm"]); + expect(argv).toContain(IMAGE); + expect(argv).toContain("--add-host=host.docker.internal:host-gateway"); + expect(argv).toContain("CBB_IN_CONTAINER=1"); + expect(argv).toContain("CBB_PLAYWRIGHT_DIR=/tmp/cbb"); + const mount = argv[argv.indexOf("-v") + 1]; + expect(mount.endsWith(":/work")).toBe(true); + const inner = argv[argv.length - 1]; + expect(inner).toContain("@playwright/test@1.61.1"); + expect(inner).toContain("--host container"); + expect(inner).toContain("--no-manifest"); + expect(inner).toContain("http://host.docker.internal:3000"); + }); + + it("rewrites 127.0.0.1 and forwards --engines into the container command", () => { + const proj = makeProject({ config: containerConfig }); + const { json } = runJson( + ["capture", "http://127.0.0.1:4173", "--dry-run", "--json", "--engines", "firefox,webkit"], + proj, + ); + expect(json.url).toBe("http://host.docker.internal:4173"); + expect(json.docker[json.docker.length - 1]).toContain("--engines firefox,webkit"); + }); + + it("does not wrap when already inside the container", () => { + const proj = makeProject({ config: containerConfig }); + // In-container + no wrap → direct capture path → Playwright install hint + // (this repo does not have @playwright/test installed). + const { exitCode, stderr } = run(["capture", "http://localhost:3000"], proj, { + env: { CBB_IN_CONTAINER: "1" }, + }); + expect(exitCode).toBe(2); + expect(stderr).toMatch(/@playwright\/test/); + expect(stderr).not.toMatch(/docker/i); + }); + + it.skipIf(playwrightResolvable)("--local bypasses the container wrapper", () => { + const proj = makeProject({ config: containerConfig }); + const { exitCode, stderr } = run(["capture", "http://127.0.0.1:9", "--local"], proj); + expect(exitCode).toBe(2); + expect(stderr).toMatch(/@playwright\/test/); + expect(stderr).not.toMatch(/docker run/i); + }); + + it("rejects an image tag it cannot derive a Playwright version from", () => { + const proj = makeProject({ + config: { + capture: { + mode: "container", + image: "mcr.microsoft.com/playwright:latest", + waitAfterLoadMs: 0, + fullPage: true, + }, + }, + }); + const { exitCode, stderr } = run(["capture", "http://localhost:3000", "--dry-run"], proj); + expect(exitCode).toBe(2); + expect(stderr).toMatch(/image/i); + }); +}); diff --git a/scripts/__tests__/regression-test.test.js b/scripts/__tests__/regression-test.test.js index 239a2db..0919f73 100644 --- a/scripts/__tests__/regression-test.test.js +++ b/scripts/__tests__/regression-test.test.js @@ -1,10 +1,13 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { execFileSync } from "child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync, readFileSync } from "fs"; import { join, dirname } from "path"; +import { tmpdir } from "os"; import { fileURLToPath } from "url"; const __dirname = dirname(fileURLToPath(import.meta.url)); const SCRIPT = join(__dirname, "..", "regression-test.sh"); +const COMMON = join(__dirname, "..", "lib", "common.sh"); function run(args = []) { try { @@ -32,3 +35,69 @@ describe("regression-test.sh — help flag", () => { expect(result.stdout).toContain("--json"); }); }); + +// The baseline directory is shared with visualBaselines (RFC 0002): committed +// firefox/webkit baselines must not surface as SKIP noise in the chromium +// regression walk, so the walk is restricted to regressionTesting.browsers. +describe("common_find_baselines — browser-filtered baseline walk", () => { + let tmp; + + beforeAll(() => { + tmp = mkdtempSync(join(tmpdir(), "find-baselines-")); + for (const rel of [ + "chromium/home/mobile_375px.png", + "chromium/about/desktop_1440px.png", + "firefox/home/mobile_375px.png", + "webkit/home/mobile_375px.png", + ]) { + const p = join(tmp, ...rel.split("/")); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, "png"); + } + writeFileSync(join(tmp, "manifest.json"), "{}"); + }); + + afterAll(() => { + rmSync(tmp, { recursive: true, force: true }); + }); + + function findBaselines(csv) { + const stdout = execFileSync( + "bash", + ["-c", `source "$1" && common_find_baselines "$2" "$3"`, "bash", COMMON, tmp, csv], + { encoding: "utf-8", timeout: 15000 }, + ); + return stdout.split("\n").filter(Boolean); + } + + it("lists only the requested browsers' PNGs, sorted", () => { + const files = findBaselines("chromium"); + expect(files).toHaveLength(2); + expect(files.every((f) => f.includes("chromium"))).toBe(true); + expect(files).toEqual([...files].sort()); + }); + + it("supports multiple browsers and ignores missing directories", () => { + const files = findBaselines("chromium, webkit, nonexistent"); + expect(files).toHaveLength(3); + expect(files.some((f) => f.includes("webkit"))).toBe(true); + expect(files.some((f) => f.includes("firefox"))).toBe(false); + }); + + it("prints nothing for an empty browser list", () => { + expect(findBaselines("")).toHaveLength(0); + }); +}); + +describe("baseline writers keep provenance in sync", () => { + it("regression-test.sh walks via common_find_baselines and syncs the manifest after --update-baselines", () => { + const source = readFileSync(SCRIPT, "utf-8"); + expect(source).toMatch(/common_find_baselines/); + expect(source).toMatch(/baseline-manifest\.js.+sync/); + }); + + it("capture-baselines.sh syncs the manifest after capture", () => { + const source = readFileSync(join(__dirname, "..", "capture-baselines.sh"), "utf-8"); + expect(source).toMatch(/baseline-manifest\.js.+sync/); + }); +}); diff --git a/scripts/__tests__/setup-baseline-lfs.test.js b/scripts/__tests__/setup-baseline-lfs.test.js new file mode 100644 index 0000000..c3eb1bc --- /dev/null +++ b/scripts/__tests__/setup-baseline-lfs.test.js @@ -0,0 +1,129 @@ +// Tests for scripts/setup-baseline-lfs.sh — Git LFS opt-in for large baseline +// sets (RFC 0002 §6.1). All mutating runs happen inside scratch git repos. +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { execFileSync } from "child_process"; +import { writeFileSync, readFileSync, mkdirSync, mkdtempSync, rmSync, existsSync } from "fs"; +import { join, dirname } from "path"; +import { tmpdir } from "os"; +import { fileURLToPath } from "url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const SCRIPT = join(__dirname, "..", "setup-baseline-lfs.sh"); + +const ATTR_LINE = ".claude/visual-qa/baselines/**/*.png filter=lfs diff=lfs merge=lfs -text"; + +let tmp; +let repoCount = 0; + +beforeAll(() => { + tmp = mkdtempSync(join(tmpdir(), "baseline-lfs-")); +}); + +afterAll(() => { + rmSync(tmp, { recursive: true, force: true }); +}); + +const gitLfsAvailable = (() => { + try { + execFileSync("git", ["lfs", "version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +})(); + +/** Scratch git repo, optionally with a visualBaselines config. */ +function makeRepo({ storage } = {}) { + const root = join(tmp, `repo-${repoCount++}`); + mkdirSync(root, { recursive: true }); + execFileSync("git", ["init", "-q"], { cwd: root }); + if (storage) { + mkdirSync(join(root, ".claude"), { recursive: true }); + writeFileSync( + join(root, ".claude", "pipeline.config.json"), + JSON.stringify({ visualBaselines: { storage } }, null, 2), + ); + } + return root; +} + +function run(args, cwd) { + try { + const stdout = execFileSync("bash", [SCRIPT, ...args], { + cwd, + encoding: "utf-8", + timeout: 30000, + }); + return { stdout, stderr: "", exitCode: 0 }; + } catch (err) { + return { stdout: err.stdout ?? "", stderr: err.stderr ?? "", exitCode: err.status }; + } +} + +describe("setup-baseline-lfs.sh", () => { + it("shows usage and exits 0 on --help", () => { + const { stdout, exitCode } = run(["--help"], tmp); + expect(exitCode).toBe(0); + expect(stdout).toContain("Usage:"); + expect(stdout).toContain("--dry-run"); + }); + + it("exits 2 outside a git repository", () => { + const bare = join(tmp, "not-a-repo"); + mkdirSync(bare, { recursive: true }); + const { exitCode, stderr } = run([], bare); + expect(exitCode).toBe(2); + expect(stderr).toMatch(/git repo/i); + }); + + it("refuses when visualBaselines.storage is not lfs", () => { + const repo = makeRepo({ storage: "git" }); + const { exitCode, stdout } = run([], repo); + expect(exitCode).toBe(1); + expect(stdout).toMatch(/storage/); + expect(stdout).toMatch(/--force/); + expect(existsSync(join(repo, ".gitattributes"))).toBe(false); + }); + + it("--dry-run prints the planned filter line without writing", () => { + const repo = makeRepo({ storage: "lfs" }); + const { exitCode, stdout } = run(["--dry-run"], repo); + expect(exitCode).toBe(0); + expect(stdout).toContain(ATTR_LINE); + expect(existsSync(join(repo, ".gitattributes"))).toBe(false); + }); + + it.skipIf(!gitLfsAvailable)("applies the filter idempotently and installs LFS locally", () => { + const repo = makeRepo({ storage: "lfs" }); + + const first = run([], repo); + expect(first.exitCode).toBe(0); + const attrs = readFileSync(join(repo, ".gitattributes"), "utf-8"); + expect(attrs).toContain(ATTR_LINE); + + // git lfs install --local writes the filter into the repo config + const filterClean = execFileSync("git", ["config", "--local", "--get", "filter.lfs.clean"], { + cwd: repo, + encoding: "utf-8", + }).trim(); + expect(filterClean).toContain("git-lfs"); + + // Migration guidance is printed but never executed + expect(first.stdout).toContain("git lfs migrate import"); + + // Second run must not duplicate the attributes line + const second = run([], repo); + expect(second.exitCode).toBe(0); + const lines = readFileSync(join(repo, ".gitattributes"), "utf-8") + .split("\n") + .filter((l) => l.includes("filter=lfs")); + expect(lines).toHaveLength(1); + }); + + it.skipIf(!gitLfsAvailable)("--force applies even when storage is git", () => { + const repo = makeRepo({ storage: "git" }); + const { exitCode } = run(["--force"], repo); + expect(exitCode).toBe(0); + expect(readFileSync(join(repo, ".gitattributes"), "utf-8")).toContain("filter=lfs"); + }); +}); diff --git a/scripts/__tests__/visual-baselines-config.test.js b/scripts/__tests__/visual-baselines-config.test.js new file mode 100644 index 0000000..a6eb7bf --- /dev/null +++ b/scripts/__tests__/visual-baselines-config.test.js @@ -0,0 +1,152 @@ +// Tests for the visualBaselines section of pipeline.config.json (RFC 0002): +// schema strictness, enums, and the structural checks the schema cannot express. +// Black-box: runs validate-pipeline-config.js as a CLI and parses --json output. +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { execFileSync } from "child_process"; +import { readFileSync, writeFileSync, mkdtempSync, rmSync } from "fs"; +import { join, dirname } from "path"; +import { tmpdir } from "os"; +import { fileURLToPath } from "url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const VALIDATOR = join(__dirname, "..", "validate-pipeline-config.js"); +const LIVE_CONFIG = join(__dirname, "..", "..", ".claude", "pipeline.config.json"); +const LIVE_SCHEMA = join(__dirname, "..", "..", ".claude", "pipeline.config.schema.json"); + +let tmp; +let liveConfig; + +beforeAll(() => { + tmp = mkdtempSync(join(tmpdir(), "vb-config-")); + liveConfig = JSON.parse(readFileSync(LIVE_CONFIG, "utf-8")); +}); + +afterAll(() => { + rmSync(tmp, { recursive: true, force: true }); +}); + +/** Run the validator against a config object written to a temp file. */ +function validate(config, name) { + const configPath = join(tmp, `${name}.json`); + writeFileSync(configPath, JSON.stringify(config, null, 2)); + let stdout; + let exitCode = 0; + try { + stdout = execFileSync( + "node", + [VALIDATOR, "--config", configPath, "--schema", LIVE_SCHEMA, "--json"], + { encoding: "utf-8", timeout: 30000 }, + ); + } catch (err) { + stdout = err.stdout; + exitCode = err.status; + } + return { ...JSON.parse(stdout), exitCode }; +} + +/** Deep-clone the live config and apply a mutator. */ +function withConfig(mutate) { + const clone = JSON.parse(JSON.stringify(liveConfig)); + mutate(clone); + return clone; +} + +describe("visualBaselines config section", () => { + it("live config is valid and includes visualBaselines (backend=commit)", () => { + const result = validate(liveConfig, "live"); + expect(result.ok).toBe(true); + expect(result.exitCode).toBe(0); + expect(liveConfig.visualBaselines).toBeDefined(); + expect(liveConfig.visualBaselines.backend).toBe("commit"); + expect(liveConfig.visualBaselines.storage).toBe("git"); + expect(liveConfig.visualBaselines.blocking).toBe(false); + expect(liveConfig.visualBaselines.threshold).toBe(liveConfig.e2e.crossBrowserDiffThreshold); + expect(liveConfig.visualBaselines.provenance.manifest).toBe( + ".claude/visual-qa/baselines/manifest.json", + ); + }); + + it("rejects unknown keys inside visualBaselines (strict schema)", () => { + const config = withConfig((c) => { + c.visualBaselines.bogusKey = true; + }); + const result = validate(config, "unknown-key"); + expect(result.ok).toBe(false); + expect(result.exitCode).toBe(1); + const flagged = result.schemaErrors.some( + (e) => e.path.startsWith("/visualBaselines") && e.extras.some((x) => x.includes("bogusKey")), + ); + expect(flagged).toBe(true); + }); + + it("rejects an unknown backend", () => { + const config = withConfig((c) => { + c.visualBaselines.backend = "s3"; + }); + const result = validate(config, "bad-backend"); + expect(result.ok).toBe(false); + expect(result.schemaErrors.some((e) => e.path === "/visualBaselines/backend")).toBe(true); + }); + + it("rejects an unknown provenance policy", () => { + const config = withConfig((c) => { + c.visualBaselines.provenance.policy = "block"; + }); + const result = validate(config, "bad-policy"); + expect(result.ok).toBe(false); + expect(result.schemaErrors.some((e) => e.path === "/visualBaselines/provenance/policy")).toBe( + true, + ); + }); + + it("accepts storage=lfs on the commit backend", () => { + const config = withConfig((c) => { + c.visualBaselines.storage = "lfs"; + }); + const result = validate(config, "lfs-ok"); + expect(result.ok).toBe(true); + }); + + it("rejects capture mode outside container|local", () => { + const config = withConfig((c) => { + c.visualBaselines.capture.mode = "vm"; + }); + const result = validate(config, "bad-mode"); + expect(result.ok).toBe(false); + expect(result.schemaErrors.some((e) => e.path === "/visualBaselines/capture/mode")).toBe(true); + }); + + it("structural: storage=lfs requires the commit backend", () => { + const config = withConfig((c) => { + c.visualBaselines.backend = "ci-artifact"; + c.visualBaselines.storage = "lfs"; + }); + const result = validate(config, "lfs-wrong-backend"); + expect(result.ok).toBe(false); + expect(result.structuralIssues.some((e) => e.path === "/visualBaselines/storage")).toBe(true); + }); + + it("structural: threshold must match e2e.crossBrowserDiffThreshold", () => { + const config = withConfig((c) => { + c.visualBaselines.threshold = 0.05; + }); + const result = validate(config, "threshold-drift"); + expect(result.ok).toBe(false); + expect( + result.structuralIssues.some( + (e) => + e.path === "/visualBaselines/threshold" && + e.message.includes("crossBrowserDiffThreshold"), + ), + ).toBe(true); + }); + + it("structural: visualBaselines.browsers must be a subset of e2e.crossBrowserBrowsers", () => { + const config = withConfig((c) => { + c.e2e.crossBrowserBrowsers = ["chromium", "firefox"]; + }); + const result = validate(config, "browser-superset"); + expect(result.ok).toBe(false); + expect(result.structuralIssues.some((e) => e.path === "/visualBaselines/browsers")).toBe(true); + }); +}); diff --git a/scripts/capture-baselines.sh b/scripts/capture-baselines.sh index 25c42e3..b5fe357 100755 --- a/scripts/capture-baselines.sh +++ b/scripts/capture-baselines.sh @@ -132,6 +132,14 @@ node "$TEMP_SCRIPT" "$URL" "$BASELINE_DIR" "$BREAKPOINTS_JSON" "$WAIT_MS" "$FULL EXIT_CODE=$? if [ $EXIT_CODE -eq 0 ]; then + # Keep RFC 0002 provenance fresh for the engines just captured (no-op when + # no manifest exists yet). + MANIFEST_PATH=$(common_config_get 'visualBaselines.provenance.manifest' '.claude/visual-qa/baselines/manifest.json') + BROWSERS_CSV=$(node -e "try { console.log(JSON.parse(process.argv[1]).join(',')); } catch { console.log(process.argv[1]); }" "$BROWSERS_JSON") + node scripts/lib/baseline-manifest.js sync \ + --baseline-dir "$BASELINE_DIR" --manifest "$MANIFEST_PATH" \ + --engines "$BROWSERS_CSV" --routes "$ROUTES" --host local > /dev/null || true + echo "" echo "=== Baselines saved to $BASELINE_DIR ===" echo "" diff --git a/scripts/cross-browser-baseline.js b/scripts/cross-browser-baseline.js new file mode 100644 index 0000000..e258f03 --- /dev/null +++ b/scripts/cross-browser-baseline.js @@ -0,0 +1,824 @@ +/** + * cross-browser-baseline.js — Cross-browser screenshot baselines (RFC 0002). + * + * Captures firefox/webkit (and chromium) baselines, verifies per-baseline + * provenance against the committed manifest, and pixel-diffs current + * screenshots against baselines via visual-diff.js at the cross-engine + * threshold. Storage is pluggable (visualBaselines.backend). + * + * Usage (prefer the ./scripts/cross-browser-baseline.sh wrapper): + * node scripts/cross-browser-baseline.js capture [url] [--local] [--engines a,b] [--json] + * node scripts/cross-browser-baseline.js compare [url] [--current-dir ] [--blocking] [--json] + * node scripts/cross-browser-baseline.js verify [--json] + * + * Config: pipeline.config.json → visualBaselines (override path via CBB_CONFIG). + * + * Exit codes: + * 0 — pass, or failures while blocking=false (Phase A non-blocking), or skip + * 1 — failures with blocking enabled, or provenance violations (verify) + * 2 — usage/environment error + */ + +import { readFileSync, writeFileSync, existsSync, mkdirSync, rmSync } from "fs"; +import { execFileSync, spawnSync } from "child_process"; +import { createRequire } from "module"; +import { join, dirname, resolve, isAbsolute } from "path"; +import { tmpdir } from "os"; +import { fileURLToPath } from "url"; + +import { + listBaselines, + parseBaselineRelPath, + recordBaselines, + verifyBaselines, + currentGitSha, + slugForRoute, +} from "./lib/baseline-manifest.js"; +import { resolveBackend, BackendError } from "./lib/baseline-backends.js"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const repoRoot = resolve(__dirname, ".."); +const VISUAL_DIFF = join(__dirname, "visual-diff.js"); + +const CROSS_ENGINES = new Set(["firefox", "webkit"]); + +class UsageError extends Error {} + +// --- Config ------------------------------------------------------------------ + +const DEFAULTS = { + enabled: true, + backend: "commit", + storage: "git", + baselineDir: ".claude/visual-qa/baselines", + screenshotDir: ".claude/visual-qa/screenshots/cross-browser", + diffDir: ".claude/visual-qa/diffs/cross-browser", + browsers: ["chromium", "firefox", "webkit"], + routes: ["/"], + breakpoints: { mobile: 375, desktop: 1440 }, + threshold: 0.03, + blocking: false, + reportFile: "cross-browser-report.md", + capture: { + mode: "container", + image: "mcr.microsoft.com/playwright:v1.61.1-noble", + waitAfterLoadMs: 1500, + fullPage: true, + }, + provenance: { + manifest: ".claude/visual-qa/baselines/manifest.json", + policy: "warn", + }, + ciArtifact: { compareAgainst: "last-green-main", retentionDays: 30 }, + service: { provider: "chromatic", projectTokenEnv: "CHROMATIC_PROJECT_TOKEN" }, +}; + +function loadCbbConfig() { + const configPath = process.env.CBB_CONFIG ?? join(repoRoot, ".claude", "pipeline.config.json"); + let vb = {}; + try { + vb = JSON.parse(readFileSync(configPath, "utf-8")).visualBaselines ?? {}; + } catch { + // missing/invalid config → defaults + } + return { + ...DEFAULTS, + ...vb, + capture: { ...DEFAULTS.capture, ...vb.capture }, + provenance: { ...DEFAULTS.provenance, ...vb.provenance }, + ciArtifact: { ...DEFAULTS.ciArtifact, ...vb.ciArtifact }, + service: { ...DEFAULTS.service, ...vb.service }, + }; +} + +function resolvePath(p) { + return isAbsolute(p) ? p : resolve(repoRoot, p); +} + +function resolveReportPath(reportFile) { + if (isAbsolute(reportFile) || reportFile.includes("/") || reportFile.includes("\\")) { + return resolvePath(reportFile); + } + return join(repoRoot, ".claude", "visual-qa", reportFile); +} + +// --- Environment ------------------------------------------------------------- + +function inContainer() { + return process.env.CBB_IN_CONTAINER === "1" || existsSync("/.dockerenv"); +} + +function detectHost() { + return inContainer() ? "container" : "local"; +} + +/** + * Playwright is not an Aurelius dependency — it belongs to the downstream app + * (or the container's scratch install). Resolve it from the most local + * package first. + */ +function resolvePlaywright() { + const candidates = [process.env.CBB_PLAYWRIGHT_DIR, process.cwd(), __dirname].filter(Boolean); + for (const dir of candidates) { + try { + const req = createRequire(join(dir, "package.json")); + const version = req("@playwright/test/package.json").version; + return { requireFrom: dir, version, req }; + } catch { + // try next candidate + } + } + return null; +} + +/** Version embedded in an mcr.microsoft.com/playwright:vX.Y.Z-distro tag. */ +export function imageTagVersion(image) { + const m = /:v(\d+\.\d+\.\d+)/.exec(image ?? ""); + return m ? m[1] : null; +} + +/** + * storage ↔ .gitattributes drift (RFC 0002 §6.1): warn when storage=lfs but + * no LFS filter covers the baseline dir, or storage=git while one does. + */ +function lfsAttributesDrift(cfg) { + let attributes = ""; + try { + attributes = readFileSync(join(repoRoot, ".gitattributes"), "utf-8"); + } catch { + // no .gitattributes — only a problem when storage=lfs + } + const covered = attributes + .split("\n") + .some((line) => line.includes(cfg.baselineDir) && line.includes("filter=lfs")); + if (cfg.storage === "lfs" && !covered) { + return ( + 'storage is "lfs" but .gitattributes has no LFS filter for the baseline dir — ' + + "run ./scripts/setup-baseline-lfs.sh" + ); + } + if (cfg.storage === "git" && covered) { + return ( + 'storage is "git" but .gitattributes routes baselines through LFS — ' + + 'set visualBaselines.storage to "lfs" or remove the filter' + ); + } + return null; +} + +// --- Output ------------------------------------------------------------------ + +function makeLogger(json) { + // With --json, stdout carries only the JSON document. + return (line) => (json ? process.stderr : process.stdout).write(`${line}\n`); +} + +function emitJson(payload) { + process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`); +} + +// --- Capture ----------------------------------------------------------------- + +async function captureTree({ url, outDir, engines, routes, breakpoints, waitMs, fullPage, log }) { + const pw = resolvePlaywright(); + if (!pw) { + throw new UsageError( + "@playwright/test is not resolvable from this project. Install it with: " + + "pnpm add -D @playwright/test && npx playwright install", + ); + } + const playwright = pw.req("@playwright/test"); + const failures = []; + for (const engine of engines) { + const launcher = playwright[engine]; + if (!launcher) { + failures.push(`${engine}: unknown engine`); + continue; + } + log(`--- ${engine} ---`); + const browser = await launcher.launch(); + try { + for (const route of routes) { + const routeSlug = slugForRoute(route); + for (const [bpName, width] of Object.entries(breakpoints)) { + const page = await browser.newPage({ viewport: { width, height: 900 } }); + try { + await page.goto(`${url}${route}`, { waitUntil: "networkidle", timeout: 30000 }); + await page.waitForTimeout(waitMs); + const dir = join(outDir, engine, routeSlug); + mkdirSync(dir, { recursive: true }); + await page.screenshot({ path: join(dir, `${bpName}_${width}px.png`), fullPage }); + log(` ✓ ${engine}/${routeSlug}/${bpName}_${width}px.png`); + } catch (err) { + failures.push(`${engine}/${routeSlug}/${bpName}: ${err.message}`); + log(` ✗ ${engine}/${routeSlug}/${bpName}: ${err.message}`); + } finally { + await page.close(); + } + } + } + } finally { + await browser.close(); + } + } + return { failures, playwrightVersion: pw.version }; +} + +/** + * Build the docker invocation that reruns this CLI's capture inside the + * pinned Playwright image (RFC 0002 §5/§6.3). The image ships browsers at + * /ms-playwright but not the npm package, so @playwright/test (at the + * version embedded in the image tag) is installed into a scratch dir that + * resolvePlaywright() picks up via CBB_PLAYWRIGHT_DIR. The repo is mounted + * at /work; localhost URLs are rewritten so the container can reach the + * host's dev server. + */ +function buildDockerCaptureCommand({ cfg, url, engines }) { + const image = cfg.capture.image; + const playwrightVersion = imageTagVersion(image); + if (!playwrightVersion) { + throw new UsageError( + `cannot derive a Playwright version from visualBaselines.capture.image "${image}" — ` + + "pin a tag like mcr.microsoft.com/playwright:v1.61.1-noble", + ); + } + const rewrittenUrl = url.replace( + /^(https?:\/\/)(localhost|127\.0\.0\.1)/, + "$1host.docker.internal", + ); + const inner = [ + "mkdir -p /tmp/cbb", + "cd /tmp/cbb", + "npm init -y >/dev/null 2>&1", + `npm i --no-fund --no-audit @playwright/test@${playwrightVersion} >/dev/null 2>&1`, + "cd /work", + `node scripts/cross-browser-baseline.js capture ${rewrittenUrl} --host container --no-manifest` + + (engines ? ` --engines ${engines.join(",")}` : ""), + ].join(" && "); + const docker = [ + "docker", + "run", + "--rm", + "-v", + `${repoRoot}:/work`, + "-w", + "/work", + "--add-host=host.docker.internal:host-gateway", + "-e", + "CBB_IN_CONTAINER=1", + "-e", + "CBB_PLAYWRIGHT_DIR=/tmp/cbb", + image, + "bash", + "-lc", + inner, + ]; + return { docker, url: rewrittenUrl, image, playwrightVersion }; +} + +async function runContainerCapture(args, cfg, log) { + const url = args.url ?? "http://localhost:3000"; + const command = buildDockerCaptureCommand({ cfg, url, engines: args.engines }); + + if (args.dryRun) { + if (args.json) emitJson({ dryRun: true, ...command }); + else { + log("Would run:"); + log(` ${command.docker.join(" ")}`); + } + return 0; + } + + try { + execFileSync("docker", ["--version"], { stdio: "ignore" }); + } catch { + throw new UsageError( + 'docker is required for container capture (visualBaselines.capture.mode = "container") — ' + + "install Docker, or pass --local for an untrusted local capture", + ); + } + + log(`Running pinned capture in ${command.image} ...`); + const spawned = spawnSync(command.docker[0], command.docker.slice(1), { + stdio: ["ignore", "inherit", "inherit"], + }); + if (spawned.status !== 0) { + throw new UsageError(`container capture failed (exit ${spawned.status ?? "unknown"})`); + } + + // Provenance is recorded host-side: the container only wrote PNGs onto the + // mounted volume, while gitSha comes from the host checkout. + const engines = args.engines ?? cfg.browsers; + const baselineDir = resolvePath(cfg.baselineDir); + const manifest = recordBaselines({ + baselineDir, + manifestPath: resolvePath(cfg.provenance.manifest), + engines, + routes: cfg.routes, + envelope: { + playwrightVersion: command.playwrightVersion, + image: command.image, + host: "container", + gitSha: currentGitSha(repoRoot), + }, + }); + + const backend = resolveBackend(cfg); + if (args.json) { + emitJson({ + captured: Object.keys(manifest.baselines).length, + host: "container", + image: command.image, + playwrightVersion: command.playwrightVersion, + backend: backend.name, + }); + } else { + log(""); + log(`Baselines captured to ${baselineDir} (host=container, ${command.image})`); + log("To persist them:"); + for (const line of backend.storeInstructions({ + baselineDir: cfg.baselineDir, + manifestPath: cfg.provenance.manifest, + })) { + log(` ${line}`); + } + } + return 0; +} + +async function cmdCapture(args, cfg) { + const log = makeLogger(args.json); + if (!cfg.enabled) { + if (args.json) emitJson({ skipped: true, reason: "visualBaselines disabled in config" }); + else log("⊘ visualBaselines disabled in config — skipping capture"); + return 0; + } + const engines = args.engines ?? cfg.browsers; + const url = args.url ?? "http://localhost:3000"; + const mode = args.local ? "local" : cfg.capture.mode; + + const drift = lfsAttributesDrift(cfg); + if (drift) log(`⚠ ${drift}`); + + if (mode === "container" && !inContainer()) { + return runContainerCapture(args, cfg, log); + } + + const host = args.host ?? detectHost(); + const baselineDir = resolvePath(cfg.baselineDir); + const localCrossEngines = engines.filter((e) => CROSS_ENGINES.has(e)); + if (host === "local" && localCrossEngines.length) { + log( + `⚠ capturing ${localCrossEngines.join("/")} outside the pinned container — ` + + "baselines will be recorded with host=local and flagged as foreign by provenance", + ); + } + + const { failures, playwrightVersion } = await captureTree({ + url, + outDir: baselineDir, + engines, + routes: cfg.routes, + breakpoints: cfg.breakpoints, + waitMs: cfg.capture.waitAfterLoadMs, + fullPage: cfg.capture.fullPage, + log, + }); + + const pinned = imageTagVersion(cfg.capture.image); + if (pinned && playwrightVersion && pinned !== playwrightVersion) { + log( + `⚠ pinned image version v${pinned} differs from resolved @playwright/test ` + + `${playwrightVersion} — update visualBaselines.capture.image or the installed Playwright`, + ); + } + + let manifest = null; + if (!args.noManifest) { + manifest = recordBaselines({ + baselineDir, + manifestPath: resolvePath(cfg.provenance.manifest), + engines, + routes: cfg.routes, + envelope: { + playwrightVersion, + image: cfg.capture.image, + host, + gitSha: currentGitSha(repoRoot), + }, + }); + } + + const backend = resolveBackend(cfg); + const instructions = backend.storeInstructions({ + baselineDir: cfg.baselineDir, + manifestPath: cfg.provenance.manifest, + }); + + if (args.json) { + emitJson({ + captured: manifest ? Object.keys(manifest.baselines).length : null, + failures, + host, + playwrightVersion, + backend: backend.name, + }); + } else { + log(""); + log(`Baselines captured to ${baselineDir} (host=${host})`); + log("To persist them:"); + for (const line of instructions) log(` ${line}`); + } + return failures.length ? 1 : 0; +} + +// --- Compare ----------------------------------------------------------------- + +function runVisualDiff(current, baseline, diffOut, threshold) { + mkdirSync(dirname(diffOut), { recursive: true }); + let stdout; + try { + stdout = execFileSync( + "node", + [ + VISUAL_DIFF, + current, + baseline, + "--output", + diffOut, + "--threshold", + String(threshold), + "--json", + ], + { encoding: "utf-8", timeout: 60000 }, + ); + } catch (err) { + stdout = err.stdout ?? ""; + } + try { + const parsed = JSON.parse(stdout); + return { + status: (parsed.status ?? "UNKNOWN").toUpperCase(), + // visual-diff.js names this "mismatchPct" but emits a 0-1 ratio. + mismatchRatio: parsed.mismatchPct ?? null, + }; + } catch { + return { status: "WARN", mismatchRatio: null }; + } +} + +function formatPct(ratio) { + return ratio != null ? `${(ratio * 100).toFixed(2)}%` : "-"; +} + +function writeCompareReport(reportPath, payload) { + const lines = [ + "# Cross-Browser Baseline Report", + "", + `**Date:** ${new Date().toISOString()}`, + `**Backend:** ${payload.backend}`, + `**Threshold:** ${payload.threshold} (${payload.threshold * 100}%)`, + `**Blocking:** ${payload.blocking}`, + `**Advisory:** ${payload.advisory ? "yes — capture envelope does not match the manifest (results are informational)" : "no"}`, + "", + "## Summary", + "", + "| Metric | Count |", + "|--------|-------|", + `| Pass | ${payload.pass} |`, + `| Fail | ${payload.fail} |`, + `| Warn | ${payload.warn} |`, + `| Skip | ${payload.skip} |`, + `| Provenance failures | ${payload.provenanceFailures} |`, + `| Provenance violations | ${payload.provenance.violations} |`, + "", + "## Results", + "", + "| Baseline | Engine | Status | Mismatch | Provenance |", + "|----------|--------|--------|----------|------------|", + ...payload.results.map( + (r) => + `| ${r.path} | ${r.engine ?? "?"} | ${r.status} | ${formatPct(r.mismatchRatio)} | ${r.provenance} |`, + ), + "", + ]; + if (payload.provenance.drift.version || payload.provenance.drift.image) { + lines.push("## Provenance drift", ""); + if (payload.provenance.drift.version) { + lines.push("- Manifest Playwright version differs from the current envelope."); + } + if (payload.provenance.drift.image) { + lines.push("- Manifest container image differs from visualBaselines.capture.image."); + } + lines.push(""); + } + mkdirSync(dirname(reportPath), { recursive: true }); + writeFileSync(reportPath, `${lines.join("\n")}\n`); +} + +/** + * service backend: the provider owns capture, diff, and review — this run + * just validates the token and hands off to the provider CLI (RFC 0002 §8). + */ +function runDelegatedCompare(args, cfg, backend, log) { + backend.requireToken(process.env); + let snapshotsFile; + if (backend.provider === "percy") { + snapshotsFile = join(tmpdir(), `cbb-percy-snapshots-${process.pid}.json`); + writeFileSync( + snapshotsFile, + backend.buildSnapshots({ url: args.url ?? "http://localhost:3000" }), + ); + } + const blocking = Boolean(args.blocking || cfg.blocking); + const argv = backend.providerArgv({ blocking, snapshotsFile }); + log(`Delegating cross-browser comparison to ${backend.provider}: ${argv.join(" ")}`); + const spawned = spawnSync(argv[0], argv.slice(1), { + stdio: args.json ? ["ignore", "inherit", "inherit"] : "inherit", + shell: process.platform === "win32", + }); + const exitCode = spawned.status ?? 2; + if (args.json) { + emitJson({ backend: "service", provider: backend.provider, delegated: true, exitCode }); + } + return exitCode; +} + +async function cmdCompare(args, cfg) { + const log = makeLogger(args.json); + if (!cfg.enabled) { + if (args.json) emitJson({ skipped: true, reason: "visualBaselines disabled in config" }); + else log("⊘ visualBaselines disabled in config — skipping compare"); + return 0; + } + + const drift = lfsAttributesDrift(cfg); + if (drift) log(`⚠ ${drift}`); + + const backend = resolveBackend(cfg); + if (backend.delegated) { + return runDelegatedCompare(args, cfg, backend, log); + } + + const engines = args.engines ?? cfg.browsers; + const fetched = await backend.fetch({ baselineDir: resolvePath(cfg.baselineDir) }); + if (!fetched.baselineRoot) { + if (args.json) emitJson({ skipped: true, reason: fetched.reason, backend: backend.name }); + else log(`⊘ ${fetched.reason}`); + return 0; + } + const baselineRoot = fetched.baselineRoot; + const manifestPath = fetched.manifestPath ?? resolvePath(cfg.provenance.manifest); + if (fetched.source) log(`Baselines fetched from ${fetched.source}`); + + const rels = listBaselines(baselineRoot, engines); + if (!rels.length) { + const reason = + "no baselines found — run ./scripts/cross-browser-baseline.sh capture " + + "(inside the pinned container) and commit the result"; + if (args.json) emitJson({ skipped: true, reason, backend: backend.name }); + else log(`⊘ ${reason}`); + return 0; + } + + const host = args.host ?? detectHost(); + const playwrightVersion = resolvePlaywright()?.version ?? null; + const provenance = verifyBaselines({ + baselineDir: baselineRoot, + manifestPath, + engines, + envelope: { playwrightVersion, host }, + image: cfg.capture.image, + }); + + let currentRoot = args.currentDir ? resolve(args.currentDir) : null; + if (!currentRoot) { + currentRoot = resolvePath(cfg.screenshotDir); + rmSync(currentRoot, { recursive: true, force: true }); + log("--- Capturing current screenshots ---"); + const { failures } = await captureTree({ + url: args.url ?? "http://localhost:3000", + outDir: currentRoot, + engines, + routes: cfg.routes, + breakpoints: cfg.breakpoints, + waitMs: cfg.capture.waitAfterLoadMs, + fullPage: cfg.capture.fullPage, + log, + }); + if (failures.length) + log(`⚠ ${failures.length} capture failure(s) — affected baselines will be skipped`); + } + + const diffRoot = resolvePath(cfg.diffDir); + rmSync(diffRoot, { recursive: true, force: true }); + + const enforce = cfg.provenance.policy === "enforce"; + const results = []; + for (const rel of rels) { + const engine = parseBaselineRelPath(rel)?.engine ?? null; + const provStatus = provenance.statuses[rel] ?? "untracked"; + if (enforce && provStatus !== "ok") { + log(`PROVENANCE: ${rel} (${provStatus}) — excluded from diff`); + results.push({ + path: rel, + engine, + status: "PROVENANCE", + mismatchRatio: null, + provenance: provStatus, + }); + continue; + } + const currentFile = join(currentRoot, ...rel.split("/")); + if (!existsSync(currentFile)) { + log(`SKIP: ${rel} (no current screenshot)`); + results.push({ + path: rel, + engine, + status: "SKIP", + mismatchRatio: null, + provenance: provStatus, + }); + continue; + } + const { status, mismatchRatio } = runVisualDiff( + currentFile, + join(baselineRoot, ...rel.split("/")), + join(diffRoot, ...rel.split("/")), + cfg.threshold, + ); + log( + `${status}: ${rel}${mismatchRatio != null ? ` (${formatPct(mismatchRatio)})` : ""}${provStatus !== "ok" ? ` [provenance: ${provStatus}]` : ""}`, + ); + results.push({ path: rel, engine, status, mismatchRatio, provenance: provStatus }); + } + + const count = (s) => results.filter((r) => r.status === s).length; + const payload = { + backend: backend.name, + engines, + threshold: cfg.threshold, + blocking: Boolean(args.blocking || cfg.blocking), + advisory: !provenance.envelopeMatch, + pass: count("PASS"), + fail: count("FAIL"), + warn: count("WARN") + count("UNKNOWN"), + skip: count("SKIP"), + provenanceFailures: count("PROVENANCE"), + provenance: { + manifestFound: provenance.manifestFound, + violations: provenance.violations, + counts: provenance.counts, + drift: provenance.drift, + envelopeMatch: provenance.envelopeMatch, + }, + results, + }; + payload.wouldBlock = payload.fail + payload.provenanceFailures > 0; + + const reportPath = resolveReportPath(cfg.reportFile); + writeCompareReport(reportPath, payload); + payload.reportPath = reportPath; + + log(""); + log( + `Pass: ${payload.pass} | Fail: ${payload.fail} | Warn: ${payload.warn} | ` + + `Skip: ${payload.skip} | Provenance: ${payload.provenanceFailures}`, + ); + if (payload.advisory) { + log("⚠ advisory: current capture envelope does not match the baseline manifest (local run?)"); + } + if (args.json) emitJson(payload); + + return payload.wouldBlock && payload.blocking ? 1 : 0; +} + +// --- Verify ------------------------------------------------------------------ + +function cmdVerify(args, cfg) { + const log = makeLogger(args.json); + if (!cfg.enabled) { + if (args.json) emitJson({ skipped: true, reason: "visualBaselines disabled in config" }); + else log("⊘ visualBaselines disabled in config — skipping verify"); + return 0; + } + const engines = args.engines ?? cfg.browsers; + const result = verifyBaselines({ + baselineDir: resolvePath(cfg.baselineDir), + manifestPath: resolvePath(cfg.provenance.manifest), + engines, + envelope: { + playwrightVersion: resolvePlaywright()?.version ?? null, + host: args.host ?? detectHost(), + }, + image: cfg.capture.image, + }); + if (args.json) { + emitJson(result); + } else { + log(`Provenance: ${result.total} baseline(s) checked, ${result.violations} violation(s)`); + for (const [rel, status] of Object.entries(result.statuses)) { + if (status !== "ok") log(` ${status}: ${rel}`); + } + if (!result.manifestFound && result.total > 0) { + log( + " no manifest — run ./scripts/cross-browser-baseline.sh capture to establish provenance", + ); + } + } + return result.violations > 0 ? 1 : 0; +} + +// --- CLI --------------------------------------------------------------------- + +function printHelp(stream = process.stdout) { + stream.write(`Usage: node scripts/cross-browser-baseline.js [options] + +Cross-browser screenshot baselines (RFC 0002). Config: pipeline.config.json → +visualBaselines (override file via CBB_CONFIG). + +Subcommands: + capture [url] Capture baselines for the configured engines and record + provenance (default url: http://localhost:3000) + compare [url] Capture current screenshots (or use --current-dir) and diff + them against stored baselines at the cross-engine threshold + verify Check baseline provenance only (no server required) + +Options: + --engines Restrict to specific engines + --json Machine-readable output on stdout + --local capture: force local capture (recorded host=local, untrusted for firefox/webkit) + --dry-run capture: print the docker command for container mode without running it + --current-dir compare: use existing screenshots instead of capturing + --blocking compare: exit 1 on failures regardless of config + --host Override detected capture host (container|local) + --no-manifest capture: skip provenance recording (internal) + -h, --help Show this message +`); +} + +function parseArgs(argv) { + const args = { json: false, local: false, blocking: false, noManifest: false }; + const positional = []; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--json") args.json = true; + else if (a === "--local") args.local = true; + else if (a === "--blocking") args.blocking = true; + else if (a === "--dry-run") args.dryRun = true; + else if (a === "--no-manifest") args.noManifest = true; + else if (a === "--engines") + args.engines = argv[++i] + ?.split(",") + .map((s) => s.trim()) + .filter(Boolean); + else if (a === "--current-dir") args.currentDir = argv[++i]; + else if (a === "--host") args.host = argv[++i]; + else if (a === "-h" || a === "--help") args.help = true; + else if (a.startsWith("--")) throw new UsageError(`Unknown option: ${a}`); + else positional.push(a); + } + args.command = positional[0]; + args.url = positional[1]; + return args; +} + +async function main() { + let args; + try { + args = parseArgs(process.argv.slice(2)); + } catch (err) { + process.stderr.write(`${err.message}\n`); + printHelp(process.stderr); + process.exit(2); + } + if (args.help || (!args.command && process.argv.slice(2).includes("--help"))) { + printHelp(); + process.exit(0); + } + if (!args.command || !["capture", "compare", "verify"].includes(args.command)) { + process.stderr.write(`Usage error: expected a subcommand (capture | compare | verify)\n`); + printHelp(process.stderr); + process.exit(2); + } + + const cfg = loadCbbConfig(); + try { + if (args.command === "capture") process.exit(await cmdCapture(args, cfg)); + if (args.command === "compare") process.exit(await cmdCompare(args, cfg)); + process.exit(cmdVerify(args, cfg)); + } catch (err) { + if (err instanceof UsageError || err instanceof BackendError) { + process.stderr.write(`✗ ${err.message}\n`); + process.exit(2); + } + process.stderr.write(`✗ Unhandled error: ${err.stack ?? err.message}\n`); + process.exit(2); + } +} + +const invokedDirect = process.argv[1] && resolve(process.argv[1]) === __filename; +if (invokedDirect) { + main(); +} diff --git a/scripts/cross-browser-baseline.sh b/scripts/cross-browser-baseline.sh new file mode 100644 index 0000000..665a8d2 --- /dev/null +++ b/scripts/cross-browser-baseline.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# cross-browser-baseline.sh — Cross-browser screenshot baselines (RFC 0002) +# +# Usage: +# ./scripts/cross-browser-baseline.sh capture [url] [--local] [--engines a,b] [--json] +# ./scripts/cross-browser-baseline.sh compare [url] [--current-dir ] [--blocking] [--json] +# ./scripts/cross-browser-baseline.sh verify [--json] +# +# Thin wrapper over scripts/cross-browser-baseline.js, which owns config +# (pipeline.config.json → visualBaselines), the provenance manifest, backend +# adapters, and the pinned-container capture. +# +# Exit codes: 0 = pass/skip, 1 = blocking failures or provenance violations, +# 2 = usage/environment error. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/common.sh +source "$SCRIPT_DIR/lib/common.sh" +cd "$(common_project_root)" + +require_cmd node "https://nodejs.org (Node.js 20+)" + +exec node "$SCRIPT_DIR/cross-browser-baseline.js" "$@" diff --git a/scripts/cross-browser-test.sh b/scripts/cross-browser-test.sh index 3d0cf10..a763388 100644 --- a/scripts/cross-browser-test.sh +++ b/scripts/cross-browser-test.sh @@ -11,6 +11,10 @@ # Results saved to .claude/visual-qa/screenshots// # # Browsers: chromium, firefox, webkit +# +# NOTE: this is an ad-hoc capture utility only — it performs no comparison. +# For baseline-backed cross-browser verification (pipeline Phase 7) use +# ./scripts/cross-browser-baseline.sh (RFC 0002). set -e diff --git a/scripts/lib/baseline-backends.js b/scripts/lib/baseline-backends.js new file mode 100644 index 0000000..ef3cc6c --- /dev/null +++ b/scripts/lib/baseline-backends.js @@ -0,0 +1,159 @@ +/** + * baseline-backends.js — RFC 0002 §8 baseline storage backend adapters. + * + * Every backend exposes the same small surface so the capture/compare CLI is + * backend-agnostic: + * + * fetch({ baselineDir }) → { baselineRoot, manifestPath?, reason? } + * make baselines available locally; baselineRoot + * null + reason = nothing to compare against yet + * storeInstructions(ctx) → string[] how new baselines are persisted + * delegated → boolean true when the provider owns capture, + * diff, and review (service backend) + * + * Backends: + * commit (default) — baselines are git-tracked in the worktree + * ci-artifact — compare against the cross-browser-baselines + * artifact published from the last green main build + * service — delegate to Chromatic/Percy (opt-in; project + * token supplied via the configured env var) + */ + +import { execFileSync } from "child_process"; +import { mkdtempSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; + +export class BackendError extends Error {} + +const ARTIFACT_NAME = "cross-browser-baselines"; +const WORKFLOW_FILE = "cross-browser-baselines.yml"; + +export function resolveBackend(config, deps = {}) { + const name = config.backend ?? "commit"; + const execFile = deps.execFile ?? ((cmd, args) => execFileSync(cmd, args, { encoding: "utf-8" })); + if (name === "commit") return commitBackend(config); + if (name === "ci-artifact") return ciArtifactBackend(config, execFile); + if (name === "service") return serviceBackend(config); + throw new BackendError( + `unknown visualBaselines.backend "${name}" (expected commit | ci-artifact | service)`, + ); +} + +function commitBackend(config) { + return { + name: "commit", + delegated: false, + async fetch({ baselineDir }) { + // Baselines are already in the worktree (git or git-lfs). + return { baselineRoot: baselineDir }; + }, + storeInstructions({ baselineDir }) { + return [ + `git add ${baselineDir}`, + `git commit -m "test: update cross-browser baselines"`, + ...(config.storage === "lfs" + ? ["(storage=lfs: ensure scripts/setup-baseline-lfs.sh has been run)"] + : []), + ]; + }, + }; +} + +function ciArtifactBackend(config, execFile) { + return { + name: "ci-artifact", + delegated: false, + async fetch() { + let runId; + try { + runId = execFile("gh", [ + "run", + "list", + "--workflow", + WORKFLOW_FILE, + "--branch", + "main", + "--status", + "success", + "--limit", + "1", + "--json", + "databaseId", + "--jq", + ".[0].databaseId", + ]).trim(); + } catch (err) { + throw new BackendError( + `the ci-artifact backend needs the gh CLI to locate the last green main build (${err.message})`, + ); + } + if (!runId) { + return { + baselineRoot: null, + reason: + "no published cross-browser baselines artifact on main yet — " + + "merge a capture to main first (the publish job in cross-browser-baselines.yml uploads it)", + }; + } + const dest = mkdtempSync(join(tmpdir(), "cbb-artifact-")); + try { + execFile("gh", ["run", "download", runId, "-n", ARTIFACT_NAME, "-D", dest]); + } catch (err) { + throw new BackendError( + `failed to download the ${ARTIFACT_NAME} artifact from run ${runId}: ${err.message}`, + ); + } + return { + baselineRoot: dest, + manifestPath: join(dest, "manifest.json"), + source: `gh run ${runId} (last green main)`, + }; + }, + storeInstructions() { + return [ + "baselines publish automatically from main (publish job in .github/workflows/cross-browser-baselines.yml)", + ]; + }, + }; +} + +function serviceBackend(config) { + const provider = config.service?.provider ?? "chromatic"; + const tokenEnv = config.service?.projectTokenEnv ?? "CHROMATIC_PROJECT_TOKEN"; + return { + name: "service", + delegated: true, + provider, + tokenEnv, + requireToken(env) { + if (!env[tokenEnv]) { + throw new BackendError( + `the ${provider} service backend needs a project token in $${tokenEnv} ` + + "(set it as a CI secret; the provider owns capture, diff, and review)", + ); + } + }, + providerArgv({ blocking, snapshotsFile } = {}) { + if (provider === "percy") { + return ["npx", "--yes", "@percy/cli", "snapshot", snapshotsFile]; + } + // Chromatic reads the project token from its env var; non-blocking runs + // mirror visualBaselines.blocking=false by exiting zero on changes. + return ["npx", "--yes", "chromatic", ...(blocking ? [] : ["--exit-zero-on-changes"])]; + }, + /** Percy full-page snapshots generated from the configured routes. */ + buildSnapshots({ url }) { + const base = url.replace(/\/$/, ""); + const routes = config.routes ?? ["/"]; + return JSON.stringify( + routes.map((route) => ({ name: route, url: `${base}${route}` })), + null, + 2, + ); + }, + storeInstructions() { + return [`baseline storage and review are owned by ${provider} — see the provider dashboard`]; + }, + }; +} diff --git a/scripts/lib/baseline-manifest.js b/scripts/lib/baseline-manifest.js new file mode 100644 index 0000000..bf9492c --- /dev/null +++ b/scripts/lib/baseline-manifest.js @@ -0,0 +1,424 @@ +/** + * baseline-manifest.js — RFC 0002 provenance manifest for cross-browser + * screenshot baselines. + * + * The manifest (default .claude/visual-qa/baselines/manifest.json, committed + * alongside the PNGs) records the capture envelope per baseline: sha256, + * engine, route, breakpoint, capture host (container|local), gitSha, plus the + * top-level Playwright version and pinned container image. The comparator + * verifies this before trusting a baseline, so stale or foreign baselines are + * flagged instead of surfacing as false pixel diffs. + * + * Library use (ESM): + * + * import { recordBaselines, verifyBaselines } from "./lib/baseline-manifest.js"; + * + * CLI use: + * + * node scripts/lib/baseline-manifest.js record [--engines a,b] [--host container] + * node scripts/lib/baseline-manifest.js sync [--engines a,b] [--host local] + * node scripts/lib/baseline-manifest.js verify [--json] + * + * Defaults come from pipeline.config.json → visualBaselines. Exit codes: + * 0 ok, 1 provenance violations (verify), 2 usage/IO error. + */ +import { readFileSync, writeFileSync, existsSync, readdirSync, mkdirSync } from "fs"; +import { execFileSync } from "child_process"; +import { createHash } from "crypto"; +import { join, dirname, resolve, relative } from "path"; +import { fileURLToPath } from "url"; +import { getValue } from "./pipeline-config.js"; + +const __filename = fileURLToPath(import.meta.url); + +/** Engines whose baselines are only trustworthy from containerized capture. */ +const CROSS_ENGINES = new Set(["firefox", "webkit"]); + +export function toPosix(p) { + return String(p).replace(/\\/g, "/"); +} + +export function sha256File(path) { + return createHash("sha256").update(readFileSync(path)).digest("hex"); +} + +/** + * Parse `//_px.png`. Breakpoint names + * may themselves contain underscores; the width is the final `_px`. + * Returns null for anything that is not a baseline PNG (e.g. manifest.json). + */ +export function parseBaselineRelPath(relPath) { + const m = toPosix(relPath).match(/^([^/]+)\/([^/]+)\/(.+)_(\d+)px\.png$/); + if (!m) return null; + return { engine: m[1], routeSlug: m[2], breakpoint: m[3], width: Number(m[4]) }; +} + +/** Route → directory slug, mirroring the capture scripts ('/' → 'home'). */ +export function slugForRoute(route) { + return route === "/" ? "home" : route.replace(/^\//, "").replace(/\//g, "-"); +} + +function routesBySlug(routes = []) { + const map = {}; + for (const route of routes) map[slugForRoute(route)] = route; + return map; +} + +export function loadManifest(manifestPath) { + if (!existsSync(manifestPath)) return null; + try { + return JSON.parse(readFileSync(manifestPath, "utf-8")); + } catch { + return null; + } +} + +function writeManifest(manifestPath, manifest) { + mkdirSync(dirname(manifestPath), { recursive: true }); + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); +} + +function sortedByKey(obj) { + return Object.fromEntries(Object.entries(obj).sort(([a], [b]) => a.localeCompare(b))); +} + +/** All baseline PNGs for one engine, as posix paths relative to baselineDir. */ +function scanEngine(baselineDir, engine) { + const results = []; + const engineRoot = join(baselineDir, engine); + if (!existsSync(engineRoot)) return results; + const walk = (dir) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.isFile() && entry.name.endsWith(".png")) { + results.push(toPosix(relative(baselineDir, full))); + } + } + }; + walk(engineRoot); + return results.sort(); +} + +/** All baseline PNGs for the given engines, posix-relative to baselineDir. */ +export function listBaselines(baselineDir, engines) { + const results = []; + for (const engine of engines) results.push(...scanEngine(baselineDir, engine)); + return results; +} + +function buildEntry(baselineDir, relPath, { host, gitSha }, slugMap) { + const parsed = parseBaselineRelPath(relPath); + return { + sha256: sha256File(join(baselineDir, relPath)), + capturedAt: new Date().toISOString(), + gitSha: gitSha ?? null, + engine: parsed?.engine ?? null, + route: parsed ? (slugMap[parsed.routeSlug] ?? null) : null, + breakpoint: parsed?.breakpoint ?? null, + host: host ?? "local", + }; +} + +/** Drop manifest entries belonging to the given engines. */ +function withoutEngines(baselines, engines) { + const next = {}; + for (const [key, entry] of Object.entries(baselines)) { + const parsed = parseBaselineRelPath(key); + if (parsed && engines.includes(parsed.engine)) continue; + next[key] = entry; + } + return next; +} + +/** + * Record provenance for the given engines from a fresh capture: entries for + * those engines are rebuilt from disk (stale ones dropped), other engines' + * entries are preserved, and the top-level envelope is updated. + */ +export function recordBaselines({ baselineDir, manifestPath, engines, envelope, routes = [] }) { + const existing = loadManifest(manifestPath) ?? {}; + const slugMap = routesBySlug(routes); + const baselines = withoutEngines(existing.baselines ?? {}, engines); + for (const engine of engines) { + for (const rel of scanEngine(baselineDir, engine)) { + baselines[rel] = buildEntry(baselineDir, rel, envelope, slugMap); + } + } + const manifest = { + playwrightVersion: envelope.playwrightVersion ?? existing.playwrightVersion ?? null, + image: envelope.image ?? existing.image ?? null, + baselines: sortedByKey(baselines), + }; + writeManifest(manifestPath, manifest); + return manifest; +} + +/** + * Refresh entries for engines whose baselines were rewritten by a sibling + * writer (capture-baselines.sh, regression-test.sh --update-baselines). + * Unchanged files keep their original entry; the top-level envelope is never + * touched. No-op (returns null) when no manifest exists yet — provenance is + * opt-in until the first cross-browser capture records one. + */ +export function syncManifest({ baselineDir, manifestPath, engines, host, gitSha, routes = [] }) { + const manifest = loadManifest(manifestPath); + if (!manifest) return null; + const slugMap = routesBySlug(routes); + const previous = manifest.baselines ?? {}; + const baselines = withoutEngines(previous, engines); + for (const engine of engines) { + for (const rel of scanEngine(baselineDir, engine)) { + const old = previous[rel]; + baselines[rel] = + old && old.sha256 === sha256File(join(baselineDir, rel)) + ? old + : buildEntry(baselineDir, rel, { host, gitSha }, slugMap); + } + } + const next = { ...manifest, baselines: sortedByKey(baselines) }; + writeManifest(manifestPath, next); + return next; +} + +/** + * Verify baselines on disk against the manifest and the current capture + * envelope. Per-baseline statuses: + * ok | untracked | modified | missing-file | foreign-host + * Manifest-level drift (version/image) and envelope match are reported + * separately: a local current envelope makes results advisory + * (envelopeMatch: false) without flagging individual baselines. + */ +export function verifyBaselines({ baselineDir, manifestPath, engines, envelope = {}, image }) { + const manifest = loadManifest(manifestPath); + const entries = manifest?.baselines ?? {}; + const statuses = {}; + + const onDisk = []; + for (const engine of engines) onDisk.push(...scanEngine(baselineDir, engine)); + + for (const rel of onDisk) { + const entry = entries[rel]; + if (!entry) { + statuses[rel] = "untracked"; + continue; + } + if (sha256File(join(baselineDir, rel)) !== entry.sha256) { + statuses[rel] = "modified"; + continue; + } + const parsed = parseBaselineRelPath(rel); + if (entry.host !== "container" && parsed && CROSS_ENGINES.has(parsed.engine)) { + statuses[rel] = "foreign-host"; + continue; + } + statuses[rel] = "ok"; + } + + const onDiskSet = new Set(onDisk); + for (const rel of Object.keys(entries)) { + const parsed = parseBaselineRelPath(rel); + if (!parsed || !engines.includes(parsed.engine)) continue; + if (!onDiskSet.has(rel)) statuses[rel] = "missing-file"; + } + + const drift = { + version: Boolean( + manifest && + envelope.playwrightVersion && + manifest.playwrightVersion && + envelope.playwrightVersion !== manifest.playwrightVersion, + ), + image: Boolean(manifest && image && manifest.image && image !== manifest.image), + }; + + const counts = { ok: 0, untracked: 0, modified: 0, missingFile: 0, foreignHost: 0 }; + const countKey = { + ok: "ok", + untracked: "untracked", + modified: "modified", + "missing-file": "missingFile", + "foreign-host": "foreignHost", + }; + for (const status of Object.values(statuses)) counts[countKey[status]]++; + + const violations = + counts.untracked + + counts.modified + + counts.missingFile + + counts.foreignHost + + (drift.version ? 1 : 0) + + (drift.image ? 1 : 0); + + const envelopeMatch = Boolean( + manifest && + envelope.host === "container" && + (!envelope.playwrightVersion || + !manifest.playwrightVersion || + envelope.playwrightVersion === manifest.playwrightVersion), + ); + + return { + manifestFound: Boolean(manifest), + statuses, + counts, + drift, + envelopeMatch, + violations, + total: Object.keys(statuses).length, + }; +} + +/** Short git sha of HEAD, or null outside a repository. */ +export function currentGitSha(cwd = process.cwd()) { + try { + return execFileSync("git", ["rev-parse", "--short", "HEAD"], { + cwd, + encoding: "utf-8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + } catch { + return null; + } +} + +// --- CLI ------------------------------------------------------------------- + +function printHelp() { + console.log(`Usage: node scripts/lib/baseline-manifest.js [options] + +Options (defaults from pipeline.config.json → visualBaselines): + --baseline-dir Baseline root + --manifest Manifest path + --engines Engines to operate on + --routes Routes for slug→route mapping + --host Capture host recorded/asserted + --playwright-version Current Playwright version (record/verify) + --image Pinned container image (record/verify) + --git-sha Override git sha (default: HEAD) + --json Machine-readable output`); +} + +function parseCliArgs(argv) { + const opts = { json: false }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--json") opts.json = true; + else if (a === "--baseline-dir") opts.baselineDir = resolve(argv[++i]); + else if (a === "--manifest") opts.manifestPath = resolve(argv[++i]); + else if (a === "--engines") + opts.engines = argv[++i] + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + else if (a === "--routes") + opts.routes = argv[++i] + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + else if (a === "--host") opts.host = argv[++i]; + else if (a === "--playwright-version") opts.playwrightVersion = argv[++i]; + else if (a === "--image") opts.image = argv[++i]; + else if (a === "--git-sha") opts.gitSha = argv[++i]; + else if (a === "-h" || a === "--help") { + printHelp(); + process.exit(0); + } else { + console.error(`Unknown argument: ${a}`); + printHelp(); + process.exit(2); + } + } + return opts; +} + +function cliMain(argv) { + const [cmd, ...rest] = argv; + if (!cmd || !["record", "sync", "verify"].includes(cmd)) { + printHelp(); + process.exit(cmd === undefined ? 2 : 2); + } + const opts = parseCliArgs(rest); + const baselineDir = + opts.baselineDir ?? + resolve(getValue("visualBaselines.baselineDir", ".claude/visual-qa/baselines")); + const manifestPath = + opts.manifestPath ?? + resolve( + getValue("visualBaselines.provenance.manifest", ".claude/visual-qa/baselines/manifest.json"), + ); + const engines = + opts.engines ?? getValue("visualBaselines.browsers", ["chromium", "firefox", "webkit"]); + const routes = opts.routes ?? getValue("visualBaselines.routes", ["/"]); + const image = opts.image ?? getValue("visualBaselines.capture.image", null); + const gitSha = opts.gitSha ?? currentGitSha(); + + if (cmd === "record") { + const manifest = recordBaselines({ + baselineDir, + manifestPath, + engines, + routes, + envelope: { + playwrightVersion: opts.playwrightVersion ?? null, + image, + host: opts.host ?? "local", + gitSha, + }, + }); + if (opts.json) console.log(JSON.stringify(manifest, null, 2)); + else + console.log( + `Recorded ${Object.keys(manifest.baselines).length} baseline(s) → ${manifestPath}`, + ); + return; + } + + if (cmd === "sync") { + const manifest = syncManifest({ + baselineDir, + manifestPath, + engines, + routes, + host: opts.host ?? "local", + gitSha, + }); + if (opts.json) console.log(JSON.stringify({ synced: Boolean(manifest) })); + else if (manifest) console.log(`Manifest synced for engines: ${engines.join(", ")}`); + else console.log("No manifest found — nothing to sync (run a cross-browser capture first)."); + return; + } + + const result = verifyBaselines({ + baselineDir, + manifestPath, + engines, + image, + envelope: { + playwrightVersion: opts.playwrightVersion ?? null, + host: opts.host ?? "local", + }, + }); + if (opts.json) { + console.log(JSON.stringify(result, null, 2)); + } else { + console.log( + `Provenance: ${result.total} baseline(s) checked, ${result.violations} violation(s)`, + ); + for (const [rel, status] of Object.entries(result.statuses)) { + if (status !== "ok") console.log(` ${status}: ${rel}`); + } + if (result.drift.version) + console.log(" drift: manifest Playwright version differs from current"); + if (result.drift.image) console.log(" drift: manifest image differs from configured image"); + if (!result.manifestFound && result.total > 0) { + console.log(" no manifest — run cross-browser-baseline.sh capture to establish provenance"); + } + } + process.exit(result.violations > 0 ? 1 : 0); +} + +const invokedDirect = process.argv[1] && resolve(process.argv[1]) === __filename; +if (invokedDirect) { + cliMain(process.argv.slice(2)); +} diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index fccc269..dca0ad5 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -137,6 +137,29 @@ common_csv_contains() { return 1 } +# --- Baseline discovery --------------------------------------------------- + +# common_find_baselines +# Print baseline PNGs (sorted) under only the given browser subdirectories. +# The baselines root is shared between regressionTesting (chromium) and +# visualBaselines (cross-browser, RFC 0002), so each consumer walks only its +# own engines instead of the whole tree. +common_find_baselines() { + local dir="$1" + local csv="$2" + local parts part + [[ -z "$csv" ]] && return 0 + IFS=',' read -ra parts <<< "$csv" + { + for part in "${parts[@]}"; do + part="$(echo "$part" | tr -d ' ')" + if [[ -n "$part" && -d "$dir/$part" ]]; then + find "$dir/$part" -name "*.png" -type f + fi + done + } | sort +} + # --- Build artifact detection -------------------------------------------- # Returns 0 if any of dist/.next/build/out exists in the cwd. diff --git a/scripts/lib/pipeline-config.js b/scripts/lib/pipeline-config.js index 1b3fe84..1b2fe31 100644 --- a/scripts/lib/pipeline-config.js +++ b/scripts/lib/pipeline-config.js @@ -1,4 +1,3 @@ -#!/usr/bin/env node /** * pipeline-config.js — Read values from .claude/pipeline.config.json * diff --git a/scripts/regression-test.sh b/scripts/regression-test.sh index 2159b3d..163413d 100755 --- a/scripts/regression-test.sh +++ b/scripts/regression-test.sh @@ -51,11 +51,15 @@ FULL_PAGE=$(common_config_get 'regressionTesting.fullPage' true) CONFIG_ROUTES_RAW=$(common_config_get 'regressionTesting.routes' '["/"]') CONFIG_ROUTES=$(node -e "try { console.log(JSON.parse(process.argv[1]).join(',')); } catch { console.log(process.argv[1]); }" "$CONFIG_ROUTES_RAW") BROWSERS_JSON=$(common_config_get 'regressionTesting.browsers' '["chromium"]') +BROWSERS_CSV=$(node -e "try { console.log(JSON.parse(process.argv[1]).join(',')); } catch { console.log(process.argv[1]); }" "$BROWSERS_JSON") REPORT_PATH=".claude/visual-qa/$REPORT_FILE" # --- Check for baselines --- -BASELINE_COUNT=$(find "$BASELINE_DIR" -name "*.png" 2>/dev/null | wc -l) +# Walk only this section's browsers: the baselines root is shared with the +# cross-browser visualBaselines tree (RFC 0002), whose firefox/webkit PNGs +# are compared by cross-browser-baseline.sh, not here. +BASELINE_COUNT=$(common_find_baselines "$BASELINE_DIR" "$BROWSERS_CSV" | wc -l) if [ "$BASELINE_COUNT" -eq 0 ]; then echo "No baseline screenshots found in $BASELINE_DIR" echo "" @@ -177,14 +181,14 @@ while IFS= read -r baseline_file; do DIFF_OUTPUT=$(node scripts/visual-diff.js "$current_file" "$baseline_file" --output "$diff_file" --threshold "$THRESHOLD" --json 2>&1) || true # Parse mismatch and status from visual-diff.js JSON output - # Fields: mismatchPct (number), status ("PASS"/"FAIL"), pass (boolean) + # Fields: mismatchPct (a 0-1 RATIO despite the name), status ("PASS"/"FAIL") PARSED=$(echo "$DIFF_OUTPUT" | node -e " let data=''; process.stdin.on('data',d=>data+=d); process.stdin.on('end',()=>{ try { const j=JSON.parse(data); - const pct = j.mismatchPct ?? '?'; + const pct = typeof j.mismatchPct === 'number' ? (j.mismatchPct * 100).toFixed(2) : '?'; const status = (j.status || 'UNKNOWN').toUpperCase(); console.log(pct + '|' + status); } catch { console.log('?|UNKNOWN'); } @@ -208,7 +212,7 @@ while IFS= read -r baseline_file; do RESULTS="$RESULTS\n| $rel_path | WARN | ${MISMATCH}% | Unable to determine status |" fi -done < <(find "$BASELINE_DIR" -name "*.png" -type f | sort) +done < <(common_find_baselines "$BASELINE_DIR" "$BROWSERS_CSV") TOTAL=$((PASS_COUNT + FAIL_COUNT + WARN_COUNT + SKIP_COUNT)) @@ -269,6 +273,12 @@ if [ "$UPDATE_BASELINES" = "true" ] && [ "$FAIL_COUNT" -eq 0 ]; then echo "Updating baselines (all tests passed)..." cp -r "$SCREENSHOT_DIR"/* "$BASELINE_DIR"/ echo "Baselines updated from current screenshots." + # Keep RFC 0002 provenance fresh for the engines this script rewrote + # (no-op when no manifest exists yet). + MANIFEST_PATH=$(common_config_get 'visualBaselines.provenance.manifest' '.claude/visual-qa/baselines/manifest.json') + node scripts/lib/baseline-manifest.js sync \ + --baseline-dir "$BASELINE_DIR" --manifest "$MANIFEST_PATH" \ + --engines "$BROWSERS_CSV" --routes "$CONFIG_ROUTES" --host local > /dev/null || true fi # --- Exit code --- diff --git a/scripts/setup-baseline-lfs.sh b/scripts/setup-baseline-lfs.sh new file mode 100644 index 0000000..eb02f28 --- /dev/null +++ b/scripts/setup-baseline-lfs.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# setup-baseline-lfs.sh — Route visual baselines through Git LFS (RFC 0002 §6.1) +# +# Usage: ./scripts/setup-baseline-lfs.sh [--dry-run] [--force] +# +# Forward-only adoption for large baseline sets (rule of thumb: >50 baselines +# or >25 MB cumulative): appends the LFS filter for the baseline directory to +# .gitattributes (idempotently) and runs `git lfs install --local`. History +# rewriting (`git lfs migrate import`) is printed as guidance only — this +# script never rewrites history. +# +# Operates on the CURRENT git repository (downstream apps run it inside their +# own repo), reading visualBaselines.storage/baselineDir from that repo's +# .claude/pipeline.config.json when present. +# +# Exit codes: 0 = done (or dry run), 1 = refused (storage is not "lfs"), +# 2 = error (not a git repo, git-lfs missing). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/common.sh +source "$SCRIPT_DIR/lib/common.sh" + +DRY_RUN=false +FORCE=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=true; shift ;; + --force) FORCE=true; shift ;; + -h|--help) + echo "Usage: setup-baseline-lfs.sh [--dry-run] [--force]" + echo "" + echo "Routes the visual baseline directory through Git LFS (RFC 0002)." + echo "Reads visualBaselines.storage from the current repo's pipeline config" + echo "and refuses unless it is \"lfs\" (override with --force)." + echo "" + echo "Options:" + echo " --dry-run Print the planned .gitattributes change without writing" + echo " --force Apply even when visualBaselines.storage is not \"lfs\"" + exit 0 + ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +if ! TARGET_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); then + say_err "not inside a git repository — run this from the project the baselines belong to" + exit 2 +fi +cd "$TARGET_ROOT" + +CONFIG_FILE=".claude/pipeline.config.json" +read_config() { + local key="$1" default="$2" + node -e " + try { + const c = JSON.parse(require('fs').readFileSync(process.argv[1], 'utf-8')); + console.log(c.visualBaselines?.[process.argv[2]] ?? process.argv[3]); + } catch { console.log(process.argv[3]); } + " "$CONFIG_FILE" "$key" "$default" 2>/dev/null || echo "$default" +} + +STORAGE=$(read_config storage git) +BASELINE_DIR=$(read_config baselineDir ".claude/visual-qa/baselines") +ATTR_LINE="$BASELINE_DIR/**/*.png filter=lfs diff=lfs merge=lfs -text" + +say_banner "Baseline Git LFS Setup" +echo "Repository: $TARGET_ROOT" +echo "Storage: $STORAGE" +echo "Filter line: $ATTR_LINE" +echo "" + +if [[ "$STORAGE" != "lfs" && "$FORCE" != "true" ]]; then + say_fail "visualBaselines.storage is \"$STORAGE\" — set it to \"lfs\" in $CONFIG_FILE first, or pass --force" + exit 1 +fi + +if [[ "$DRY_RUN" == "true" ]]; then + say_step "dry run — would append to .gitattributes:" + echo " $ATTR_LINE" + say_step "dry run — would run: git lfs install --local" + exit 0 +fi + +if ! git lfs version >/dev/null 2>&1; then + say_err "git-lfs is not installed (https://git-lfs.com) — install it and re-run" + exit 2 +fi + +if [[ -f .gitattributes ]] && grep -qF "$ATTR_LINE" .gitattributes; then + say_skip ".gitattributes already contains the baseline LFS filter" +else + # Guard against appending onto a final line with no trailing newline. + if [[ -f .gitattributes && -s .gitattributes && -n "$(tail -c1 .gitattributes)" ]]; then + echo "" >> .gitattributes + fi + echo "$ATTR_LINE" >> .gitattributes + say_pass "added baseline LFS filter to .gitattributes" +fi + +git lfs install --local > /dev/null +say_pass "git lfs install --local complete" + +echo "" +say_banner "Next Steps" +echo "1. Commit the attributes change:" +echo " git add .gitattributes" +echo " git commit -m 'chore: route visual baselines through git-lfs'" +echo "2. Baselines added or updated from now on are stored in LFS (forward-only)." +echo "3. Optional — rewrite EXISTING history into LFS (destructive, rewrites" +echo " commits; coordinate with anyone who has clones):" +echo " git lfs migrate import --include=\"$BASELINE_DIR/**/*.png\" --everything" +echo " This script never runs the migration itself." diff --git a/scripts/validate-pipeline-config.js b/scripts/validate-pipeline-config.js index 778296b..13ff5f9 100644 --- a/scripts/validate-pipeline-config.js +++ b/scripts/validate-pipeline-config.js @@ -94,6 +94,8 @@ function formatSchemaError(err) { * - caching.phaseInputs keys reference real inputCategories * - storybook.viewports reference real visualDiff.breakpoints * - e2e.requiredBreakpoints reference real visualDiff.breakpoints + * - visualBaselines: storage=lfs only on the commit backend; threshold + * equals e2e.crossBrowserDiffThreshold; browsers ⊆ e2e.crossBrowserBrowsers */ function structuralChecks(config) { const issues = []; @@ -179,7 +181,41 @@ function structuralChecks(config) { } } - // 4. qualityGate.mutationScore.threshold vs mutationTesting.scoreThreshold drift + // 4. visualBaselines cross-field rules (RFC 0002) + const vb = config.visualBaselines; + if (vb) { + if (vb.storage === "lfs" && vb.backend !== undefined && vb.backend !== "commit") { + issues.push({ + path: "/visualBaselines/storage", + message: `"lfs" applies to the commit backend only (backend is "${vb.backend}")`, + }); + } + const crossThreshold = config.e2e?.crossBrowserDiffThreshold; + if ( + vb.threshold !== undefined && + crossThreshold !== undefined && + vb.threshold !== crossThreshold + ) { + issues.push({ + path: "/visualBaselines/threshold", + message: `disagrees with /e2e/crossBrowserDiffThreshold (${vb.threshold} vs ${crossThreshold}) — a single cross-engine tolerance must apply`, + }); + } + const crossBrowsers = config.e2e?.crossBrowserBrowsers; + if (vb.browsers && crossBrowsers) { + const allowed = new Set(crossBrowsers); + for (const b of vb.browsers) { + if (!allowed.has(b)) { + issues.push({ + path: "/visualBaselines/browsers", + message: `"${b}" is not in /e2e/crossBrowserBrowsers — baselines would be captured for an engine the E2E matrix never exercises`, + }); + } + } + } + } + + // 5. qualityGate.mutationScore.threshold vs mutationTesting.scoreThreshold drift const gateThreshold = config.qualityGate?.mutationScore?.threshold; const runnerThreshold = config.mutationTesting?.scoreThreshold; if (