Skip to content

Commit 9a4aa64

Browse files
kyle-ssgtalissoncosta
authored andcommitted
Visual regression e2e
1 parent d0b0afc commit 9a4aa64

18 files changed

Lines changed: 315 additions & 18 deletions

.github/workflows/.reusable-docker-e2e-tests.yml

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -110,12 +110,13 @@ jobs:
110110
working-directory: frontend
111111
run: make test
112112
env:
113-
opts: ${{ inputs.args }}${{ inputs.visual-regression-update && ' --update-snapshots' || '' }}
113+
opts: ${{ inputs.args }}
114114
API_IMAGE: ${{ inputs.api-image }}
115115
E2E_IMAGE: ${{ inputs.e2e-image }}
116116
E2E_CONCURRENCY: ${{ inputs.concurrency }}
117117
E2E_RETRIES: 2
118118
VISUAL_REGRESSION: ${{ inputs.visual-regression && '1' || '' }}
119+
VISUAL_REGRESSION_ARGS: ${{ inputs.visual-regression-update && '--update-snapshots' || '' }}
119120
SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
120121
GITHUB_ACTION_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
121122
timeout-minutes: 20
@@ -195,35 +196,43 @@ jobs:
195196
append: true
196197
message: ${{ steps.report-summary-success.outputs.summary || steps.report-summary-failure.outputs.summary }}
197198

198-
# Visual regression: upload baselines (for main) or report (for PRs)
199-
- name: Upload visual regression baselines
200-
if: always() && inputs.visual-regression
199+
# Visual regression: after all E2E retries, run comparison and upload results
200+
- name: Upload visual regression baselines (main branch)
201+
if: always() && inputs.visual-regression-update
201202
uses: actions/upload-artifact@v4
202203
with:
203204
name: visual-regression-baselines
204-
path: frontend/e2e/visual-regression-snapshots/
205+
path: frontend/e2e/visual-regression-screenshots/
205206
retention-days: 90
206207
overwrite: true
207208

209+
- name: Upload visual regression report
210+
if: always() && inputs.visual-regression && !inputs.visual-regression-update
211+
uses: actions/upload-artifact@v4
212+
with:
213+
name: visual-regression-report-${{ github.run_id }}-${{ strategy.job-index }}
214+
path: frontend/e2e/visual-regression-report/
215+
retention-days: 30
216+
208217
- name: Generate visual regression summary
209-
if: always() && inputs.visual-regression && github.event_name == 'pull_request'
218+
if: always() && inputs.visual-regression && !inputs.visual-regression-update && github.event_name == 'pull_request'
210219
id: visual-regression-summary
211220
shell: bash
212221
run: |
213222
if [ "${{ steps.download-baseline.outcome }}" != "success" ]; then
214223
echo "message=No baseline found — first run. Baselines will be generated after merge to main." >> $GITHUB_OUTPUT
215224
else
216-
DIFF_COUNT=$(find frontend/e2e/test-results -name "*-diff.png" 2>/dev/null | wc -l | tr -d ' ')
217-
BASELINE_COUNT=$(find frontend/e2e/visual-regression-snapshots -name "*.png" 2>/dev/null | wc -l | tr -d ' ')
218-
if [ "$DIFF_COUNT" = "0" ]; then
219-
echo "message=No visual changes detected ($BASELINE_COUNT screenshots matched)." >> $GITHUB_OUTPUT
225+
SCREENSHOT_COUNT=$(find frontend/e2e/visual-regression-screenshots -name "*.png" 2>/dev/null | wc -l | tr -d ' ')
226+
REPORT_EXISTS=$(test -d frontend/e2e/visual-regression-report && echo "true" || echo "false")
227+
if [ "$REPORT_EXISTS" = "true" ]; then
228+
echo "message=$SCREENSHOT_COUNT screenshots compared. See report for details." >> $GITHUB_OUTPUT
220229
else
221-
echo "message=$DIFF_COUNT visual change(s) detected. See HTML report for diff images." >> $GITHUB_OUTPUT
230+
echo "message=$SCREENSHOT_COUNT screenshots captured but comparison did not run." >> $GITHUB_OUTPUT
222231
fi
223232
fi
224233
225234
- name: Comment PR with visual regression results
226-
if: always() && inputs.visual-regression && github.event_name == 'pull_request' && steps.visual-regression-summary.outputs.message
235+
if: always() && inputs.visual-regression && !inputs.visual-regression-update && github.event_name == 'pull_request' && steps.visual-regression-summary.outputs.message
227236
uses: marocchino/sticky-pull-request-comment@v2
228237
with:
229238
header: visual-regression-results

frontend/.gitignore

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ e2e/test-results/
3737
*storybook.log
3838
storybook-static
3939

40-
# Visual regression snapshots (stored as CI artifacts, not in git)
40+
# Visual regression (baselines stored as CI artifacts, not in git)
4141
e2e/visual-regression-snapshots/
4242
e2e/visual-regression-screenshots/
43+
e2e/visual-regression-report/
44+
e2e/tests/_visual-regression-compare.pw.ts
45+
e2e/tests/_visual-regression-compare.pw.ts-snapshots/

frontend/Makefile

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,16 @@ serve:
3131
test:
3232
@echo "Running E2E tests..."
3333
@docker compose run --name e2e-test-run frontend \
34-
sh -c 'npx cross-env E2E_CONCURRENCY=${E2E_CONCURRENCY} E2E_RETRIES=${E2E_RETRIES} npm run test -- $(opts)' \
34+
sh -c 'npx cross-env E2E_CONCURRENCY=${E2E_CONCURRENCY} E2E_RETRIES=${E2E_RETRIES} npm run test -- $(opts); \
35+
EXIT=$$?; \
36+
if [ "$${VISUAL_REGRESSION}" = "1" ]; then npm run test:visual:compare -- $${VISUAL_REGRESSION_ARGS} || true; fi; \
37+
exit $$EXIT' \
3538
|| TEST_FAILED=1; \
3639
echo "Copying test results from container..."; \
3740
docker cp e2e-test-run:/srv/flagsmith/e2e/test-results ./e2e/test-results 2>/dev/null || echo "No test results to copy"; \
3841
docker cp e2e-test-run:/srv/flagsmith/e2e/playwright-report ./e2e/playwright-report 2>/dev/null || echo "No HTML report to copy"; \
42+
docker cp e2e-test-run:/srv/flagsmith/e2e/visual-regression-screenshots ./e2e/visual-regression-screenshots 2>/dev/null || echo "No visual regression screenshots to copy"; \
43+
docker cp e2e-test-run:/srv/flagsmith/e2e/visual-regression-report ./e2e/visual-regression-report 2>/dev/null || echo "No visual regression report to copy"; \
3944
docker rm e2e-test-run 2>/dev/null || true; \
4045
if [ "$$TEST_FAILED" = "1" ]; then \
4146
echo "\n=== API logs ===" && docker compose logs flagsmith-api && \

frontend/README.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,32 @@ E2E_RETRIES=0 SKIP_BUNDLE=1 E2E_CONCURRENCY=1 npm run test -- tests/flag-tests.p
145145
- `trace.zip` - Interactive trace viewer
146146
- Screenshots and videos
147147

148+
#### Visual Regression
149+
150+
Visual regression screenshots are captured during E2E tests via `visualSnapshot()` calls. They are a no-op unless `VISUAL_REGRESSION=1` is set. Comparison runs as a separate step after all E2E retries complete, so flaky tests don't affect the report.
151+
152+
```bash
153+
# 1. Run E2E tests with screenshot capture (with retries)
154+
VISUAL_REGRESSION=1 npm run test
155+
156+
# 2a. Generate/update baselines from captured screenshots
157+
npm run test:visual:compare -- --update-snapshots
158+
159+
# 2b. Compare screenshots against baselines (generates Playwright report with diffs)
160+
npm run test:visual:compare
161+
162+
# 3. Open the report
163+
npm run test:visual:report
164+
```
165+
166+
Visual diffs never fail CI — they are reported via PR comment and the Playwright HTML report.
167+
168+
Screenshots are saved to `e2e/visual-regression-screenshots/`, baselines to `e2e/visual-regression-snapshots/` (both git-ignored). In CI, the main branch uploads screenshots as baseline artifacts, and PRs download them for comparison.
169+
170+
| Variable | Description |
171+
|----------|-------------|
172+
| `VISUAL_REGRESSION=1` | Enable screenshot capture during E2E tests |
173+
148174
#### Claude Code Commands
149175

150176
When using Claude Code, these commands are available for e2e testing:
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import * as fs from 'fs'
2+
import * as path from 'path'
3+
4+
const BASELINES_DIR = path.resolve(__dirname, 'visual-regression-snapshots')
5+
const SCREENSHOTS_DIR = path.resolve(__dirname, 'visual-regression-screenshots')
6+
const COMPARE_TEST_FILE = path.resolve(__dirname, 'tests', '_visual-regression-compare.pw.ts')
7+
8+
/**
9+
* Generates a Playwright test file that compares each captured screenshot
10+
* against its baseline using toMatchSnapshot(). Run this AFTER E2E tests
11+
* complete to get a Playwright HTML report with diff viewer.
12+
*
13+
* Screenshots and baselines use the same flat naming convention:
14+
* {testFileName}--{snapshotName}.png (dots replaced with dashes)
15+
* e.g. flag-tests-pw-ts--features-list.png
16+
*/
17+
18+
if (!fs.existsSync(SCREENSHOTS_DIR)) {
19+
console.log('No screenshots found — run E2E tests with VISUAL_REGRESSION=1 first.')
20+
process.exit(0)
21+
}
22+
23+
// Collect screenshots
24+
const screenshots = fs
25+
.readdirSync(SCREENSHOTS_DIR)
26+
.filter((f) => f.endsWith('.png'))
27+
28+
if (screenshots.length === 0) {
29+
console.log('No screenshots to compare.')
30+
process.exit(0)
31+
}
32+
33+
if (!fs.existsSync(BASELINES_DIR)) {
34+
fs.mkdirSync(BASELINES_DIR, { recursive: true })
35+
}
36+
37+
// Build test entries from all screenshots
38+
const pairs: { file: string; label: string }[] = []
39+
for (const png of screenshots) {
40+
const label = png
41+
.replace('.png', '')
42+
.replace(/^(.+?)--(.+)$/, (_, testFile, name) => {
43+
const restored = testFile.replace(/-pw-ts$/, '.pw.ts').replace(/-/g, '.')
44+
return `${restored} / ${name.replace(/-/g, ' ')}`
45+
})
46+
pairs.push({ file: png, label })
47+
}
48+
49+
// Generate Playwright test file
50+
const testCases = pairs
51+
.map(({ file, label }) => {
52+
const screenshotPath = path.join(SCREENSHOTS_DIR, file).replace(/\\/g, '\\\\').replace(/'/g, "\\'")
53+
return `
54+
test('${label}', async () => {
55+
const screenshot = fs.readFileSync('${screenshotPath}')
56+
expect(screenshot).toMatchSnapshot('${file}', {
57+
maxDiffPixels: 300,
58+
threshold: 0.02,
59+
})
60+
})`
61+
})
62+
.join('\n')
63+
64+
const testContent = `// Auto-generated by compare-visual-regression.ts — do not edit
65+
import { test, expect } from '@playwright/test'
66+
import * as fs from 'fs'
67+
68+
test.describe('Visual Regression', () => {
69+
${testCases}
70+
})
71+
`
72+
73+
fs.writeFileSync(COMPARE_TEST_FILE, testContent)
74+
console.log(`Generated ${pairs.length} comparison tests → ${COMPARE_TEST_FILE}`)
Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
import { expect, Page, TestInfo } from '@playwright/test'
2+
import * as fs from 'fs'
3+
import * as path from 'path'
4+
5+
/**
6+
* CSS injected before every visual snapshot to hide dynamic content
7+
* that changes between runs. Playwright's toHaveScreenshot() already
8+
* handles animations (animations: 'disabled') and caret (caret: 'hide'),
9+
* so we only target app-specific volatile elements here.
10+
*/
11+
const STABILISING_CSS = `
12+
/* Hide environment select (contains dynamic API key) */
13+
#environment-select {
14+
visibility: hidden !important;
15+
}
16+
17+
/* Hide timestamps and relative dates */
18+
.ago,
19+
time,
20+
[data-test*="timestamp"],
21+
[data-test*="ago"],
22+
.text-muted:has(> .ago),
23+
.relative-date {
24+
visibility: hidden !important;
25+
}
26+
27+
/* Hide loading spinners */
28+
.spinner,
29+
.loader,
30+
[class*="spinner"],
31+
[class*="loader"] {
32+
display: none !important;
33+
}
34+
35+
/* Hide any live chat / support widgets */
36+
.intercom-launcher,
37+
#intercom-container,
38+
.drift-widget,
39+
[class*="chatbot"],
40+
iframe[title*="chat"],
41+
iframe[title*="Chat"] {
42+
display: none !important;
43+
}
44+
45+
/* Stabilise scrollbars across platforms */
46+
::-webkit-scrollbar {
47+
display: none !important;
48+
}
49+
* {
50+
scrollbar-width: none !important;
51+
}
52+
`
53+
54+
/** Directory where screenshots are captured during E2E runs */
55+
const SCREENSHOTS_DIR = path.resolve(process.cwd(), 'e2e', 'visual-regression-screenshots')
56+
57+
/** Directory where baselines live (downloaded from main in CI) */
58+
const BASELINES_DIR = path.resolve(process.cwd(), 'e2e', 'visual-regression-snapshots')
59+
60+
/**
61+
* Whether visual regression snapshots are enabled for this run.
62+
*/
63+
export function isVisualRegressionEnabled(): boolean {
64+
return process.env.VISUAL_REGRESSION === '1'
65+
}
66+
67+
/**
68+
* Wait for the page to settle before taking a screenshot.
69+
*/
70+
async function preparePage(page: Page): Promise<void> {
71+
await page.addStyleTag({ content: STABILISING_CSS })
72+
73+
// Wait for images to finish loading
74+
await page
75+
.evaluate(() => {
76+
return Promise.all(
77+
Array.from(document.images)
78+
.filter((img) => !img.complete)
79+
.map(
80+
(img) =>
81+
new Promise((resolve) => {
82+
img.addEventListener('load', resolve)
83+
img.addEventListener('error', resolve)
84+
setTimeout(resolve, 5000)
85+
}),
86+
),
87+
)
88+
})
89+
.catch(() => {})
90+
91+
// Double rAF to ensure paint is complete
92+
await page.evaluate(() => {
93+
return new Promise((resolve) => {
94+
requestAnimationFrame(() => {
95+
requestAnimationFrame(() => {
96+
resolve(undefined)
97+
})
98+
})
99+
})
100+
})
101+
102+
// Small settle time for any final layout shifts
103+
await page.waitForTimeout(500)
104+
}
105+
106+
/**
107+
* Take a screenshot during E2E tests and save it to the screenshots directory.
108+
*
109+
* This ONLY captures the screenshot — it does NOT compare against baselines.
110+
* Comparison happens as a separate step after all E2E retries have completed,
111+
* via `npx tsx e2e/compare-visual-regression.ts`.
112+
*
113+
* @param page Playwright page
114+
* @param name Descriptive snapshot name, e.g. "features-list"
115+
* @param testInfo Playwright testInfo for resolving the test file name
116+
*/
117+
export async function visualSnapshot(
118+
page: Page,
119+
name: string,
120+
testInfo: TestInfo,
121+
): Promise<void> {
122+
if (!isVisualRegressionEnabled()) return
123+
124+
await preparePage(page)
125+
126+
// Save with the sanitised name Playwright's toMatchSnapshot expects:
127+
// {testFileName}--{name} with dots replaced by dashes
128+
const testFileName = path.basename(testInfo.file)
129+
const sanitisedName = `${testFileName}--${name}`.replace(/\./g, '-')
130+
fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true })
131+
132+
const screenshotPath = path.join(SCREENSHOTS_DIR, `${sanitisedName}.png`)
133+
await page.screenshot({
134+
path: screenshotPath,
135+
fullPage: true,
136+
animations: 'disabled',
137+
caret: 'hide',
138+
scale: 'css',
139+
})
140+
}

frontend/e2e/tests/environment-test.pw.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ test.describe('Environment Tests', () => {
1111
login,
1212
setText,
1313
waitForElementVisible,
14+
waitForToastsToClear,
1415
} = createHelpers(page);
1516

1617
log('Login')
@@ -20,6 +21,7 @@ test.describe('Environment Tests', () => {
2021
log('Create environment')
2122
await click('#create-env-link')
2223
await createEnvironment('Staging')
24+
await waitForToastsToClear()
2325
await visualSnapshot(page, 'environment-created', testInfo)
2426

2527
log('Edit Environment')

frontend/e2e/tests/flag-tests.pw.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ test.describe('Flag Tests', () => {
1919
waitForElementClickable,
2020
waitForElementVisible,
2121
waitForFeatureSwitch,
22+
waitForToastsToClear,
2223
} = createHelpers(page);
2324

2425
log('Login')
@@ -51,6 +52,7 @@ test.describe('Flag Tests', () => {
5152
{ value: 'small', weight: 0 },
5253
]})
5354

55+
await waitForToastsToClear()
5456
await visualSnapshot(page, 'features-list', testInfo)
5557

5658
log('Create Short Life Feature')

frontend/e2e/tests/segment-test.pw.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ test('Segment test 1 - Create, update, and manage segments with multivariate fla
114114
await assertInputValue(byId(`rule-${0}-value-0`), `${lastRule.value + 1}`)
115115
await deleteSegmentFromPage('segment_to_update')
116116

117+
await waitForToastsToClear()
117118
await visualSnapshot(page, 'segments-list', testInfo)
118119

119120
log('Create segment')

0 commit comments

Comments
 (0)