visual-regression.yml

name: Visual regression

on: [pull_request]

concurrency:
  group: gh-pages
  cancel-in-progress: false

jobs:
  visual-regression:
    name: Capture, diff, and publish report
    runs-on: ubuntu-latest
    container:
      image: cypress/browsers:latest
      options: --user 1001
    permissions:
      contents: write
      pull-requests: write
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - uses: pnpm/action-setup@v6
      - uses: actions/setup-node@v6
        with:
          node-version: '24'
          cache: 'pnpm'

      - name: Cache Cypress binary
        uses: actions/cache@v5
        with:
          path: ~/.cache/Cypress
          key: ${{ runner.os }}-cypress-${{ hashFiles('regression-test/package-lock.json') }}

      - name: Build InstUI
        run: pnpm install --frozen-lockfile && pnpm run bootstrap

      - name: Install regression-test dependencies
        run: npm ci
        working-directory: regression-test

      - name: Install Cypress binary
        run: npx cypress install
        working-directory: regression-test

      - name: Fetch baselines
        run: |
          mkdir -p regression-test/.baselines
          if git fetch origin visual-baselines 2>/dev/null; then
            git worktree add /tmp/baselines-wt origin/visual-baselines
            cp -r /tmp/baselines-wt/screenshots/. regression-test/.baselines/ 2>/dev/null || echo "::warning::baselines branch has no screenshots/ dir"
            git worktree remove --force /tmp/baselines-wt
          else
            echo "::warning::visual-baselines branch does not exist — first run, everything will be 'added'"
          fi

      - name: Run Cypress
        id: cypress
        uses: cypress-io/github-action@v7
        continue-on-error: true
        env:
          ELECTRON_EXTRA_LAUNCH_ARGS: "--remote-debugging-port=9222"
        with:
          install: false
          build: npm run build
          start: npm start
          working-directory: regression-test

      - name: Flatten screenshots
        run: |
          mkdir -p regression-test/.actual
          find regression-test/cypress/screenshots -name '*.png' -exec cp {} regression-test/.actual/ \;

      - name: Diff and generate report
        id: diff
        working-directory: regression-test
        run: >
          npx ui-scripts visual-diff
          --actual-dir .actual
          --baseline-dir .baselines
          --output-dir .report
          --no-fail-on-missing-baseline
          --pr-number ${{ github.event.pull_request.number }}
          --pr-url ${{ github.event.pull_request.html_url }}
          --meta cypress/meta.json
          --source-base-url https://github.com/${{ github.repository }}/blob/${{ github.head_ref }}/regression-test/src/app
        continue-on-error: true

      - name: Publish report to gh-pages
        uses: peaceiris/actions-gh-pages@v4
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: regression-test/.report
          publish_branch: gh-pages
          destination_dir: visual-regression/pr-${{ github.event.pull_request.number }}
          keep_files: true
          commit_message: "visual-regression: PR #${{ github.event.pull_request.number }} (${{ github.sha }})"

      - name: Read summary
        id: summary
        env:
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          node -e "
            const fs = require('fs');
            const { summary: s, results } = JSON.parse(fs.readFileSync('regression-test/.report/summary.json', 'utf8'));
            const base = 'https://instructure.design/visual-regression/pr-' + process.env.PR_NUMBER;
            const interesting = results.filter(r => r.status === 'changed' || r.status === 'removed');
            let md = '';
            if (interesting.length) {
              md = '<details>\n<summary>Diff images (' + interesting.length + ')</summary>\n\n';
              for (const r of interesting) {
                if (r.status === 'changed') {
                  md += '#### ' + r.name + ' — ' + r.numDiff + ' pixels differ\n\n';
                  md += '![](' + base + '/diff/' + r.name + ')\n\n';
                } else {
                  md += '#### ' + r.name + ' — baseline no longer produced\n\n';
                }
              }
              md += '</details>';
            }
            const lines = [
              'total=' + s.total,
              'unchanged=' + s.unchanged,
              'changed=' + s.changed,
              'added=' + s.added,
              'removed=' + s.removed,
              'status=' + (s.changed > 0 || s.removed > 0 ? '⚠️ **Changes detected.**' : '✅ **No changes.**')
            ];
            let out = lines.join('\n') + '\n';
            out += 'diff_images<<DIFFIMAGES_EOF\n' + md + '\nDIFFIMAGES_EOF\n';
            fs.appendFileSync(process.env.GITHUB_OUTPUT, out);
          "

      - name: Post PR comment
        uses: marocchino/sticky-pull-request-comment@v3
        with:
          header: visual-regression
          message: |
            ## Visual regression report

            ${{ steps.summary.outputs.status }}

            | Status | Count |
            |---|---|
            | Unchanged | ${{ steps.summary.outputs.unchanged }} |
            | Changed | ${{ steps.summary.outputs.changed }} |
            | New | ${{ steps.summary.outputs.added }} |
            | Removed | ${{ steps.summary.outputs.removed }} |

            📊 **[View full report](https://instructure.design/visual-regression/pr-${{ github.event.pull_request.number }}/index.html)**

            ${{ steps.summary.outputs.diff_images }}

            <sub>Baselines come from the `visual-baselines` branch. They refresh on every merge to `master`.</sub>

      - name: Fail if regressions or Cypress failed
        if: steps.diff.outcome == 'failure' || steps.cypress.outcome == 'failure'
        run: exit 1