Skip to content

Commit 1383cb8

Browse files
Test Userwarp-agent
Test User
and
warp-agent
committed
Update e2e test suite with improved configuration and additional tests
- Updated playwright configuration and test utilities - Modified existing e2e test specs for better coverage - Added browser configuration and mobile smoke tests Co-Authored-By: Warp <agent@warp.dev>
1 parent fe984f7 commit 1383cb8

15 files changed

Lines changed: 1291 additions & 52 deletions

‎tests/e2e/browser-config.ts‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
/**
2+
* Browser-Specific Configuration for E2E Tests
3+
*
4+
* This module centralizes all browser-specific settings, timeouts, and quirk flags.
5+
* Import these configurations in test utilities and test files to handle cross-browser
6+
* differences consistently.
7+
*
8+
* Key browser differences addressed:
9+
* - Firefox: Slower CSS rendering, NS_BINDING_ABORTED errors during navigation
10+
* - WebKit: Delayed element rendering, localStorage timing issues
11+
* - Mobile: Touch events required, smaller viewports need scroll handling
12+
*/
13+
14+
/**
15+
* Browser-specific timeout configurations
16+
*
17+
* Chromium is the baseline. Other browsers have multipliers applied based on
18+
* observed performance characteristics.
19+
*
20+
* - Firefox: +50% for CSS rendering and form validation
21+
* - WebKit: +40% for element stabilization and animations
22+
* - Mobile: +50% for touch event registration and viewport adjustments
23+
*/
24+
export const BROWSER_TIMEOUTS = {
25+
chromium: {
26+
action: 10000, // Default actionTimeout
27+
expect: 5000, // Default expect timeout
28+
navigation: 30000, // Page navigation timeout
29+
formValidation: 3000,
30+
animation: 500,
31+
},
32+
firefox: {
33+
action: 15000, // +50% for slower CSS rendering
34+
expect: 8000, // +60% for async form validation
35+
navigation: 45000, // +50% for network handling
36+
formValidation: 5000, // Firefox renders validation messages asynchronously
37+
animation: 800, // CSS transitions take longer
38+
},
39+
webkit: {
40+
action: 14000, // +40% for element stabilization
41+
expect: 7000, // +40% for delayed rendering
42+
navigation: 40000, // +33% for Safari's network stack
43+
formValidation: 4000,
44+
animation: 700, // WebKit animation timing differences
45+
},
46+
mobile: {
47+
action: 15000, // +50% for touch event registration
48+
expect: 10000, // +100% for viewport stabilization
49+
navigation: 60000, // +100% for mobile network handling
50+
formValidation: 5000,
51+
animation: 1000, // Mobile animations may be slower
52+
},
53+
} as const;
54+
55+
/**
56+
* Browser quirk flags
57+
*
58+
* These flags indicate which workarounds are needed for each browser.
59+
* Use these to conditionally apply browser-specific handling in tests.
60+
*/
61+
export const BROWSER_QUIRKS = {
62+
firefox: {
63+
/** Firefox needs extra wait for async form validation rendering */
64+
needsFormValidationWait: true,
65+
/** Firefox's NS_BINDING_ABORTED error during navigation is benign */
66+
hasNSBindingAborted: true,
67+
/** Firefox may need reducedMotion for consistent animation timing */
68+
needsReducedMotion: true,
69+
/** Firefox click events may need explicit wait for element stability */
70+
needsClickStability: false,
71+
/** Firefox localStorage is synchronous but needs reload for visibility */
72+
hasDelayedLocalStorage: false,
73+
},
74+
webkit: {
75+
/** WebKit elements may not be stable immediately after appearing */
76+
needsElementStabilityWait: true,
77+
/** WebKit localStorage writes may not be immediately readable */
78+
hasDelayedLocalStorage: true,
79+
/** WebKit forms need click-then-fill pattern for reliable input */
80+
needsClickBeforeFill: true,
81+
/** WebKit animations need explicit completion wait */
82+
needsAnimationWait: true,
83+
/** WebKit needs extra time after navigation for DOM stability */
84+
needsPostNavigationWait: true,
85+
},
86+
mobile: {
87+
/** Mobile browsers require touch events instead of mouse clicks */
88+
needsTouchEvents: true,
89+
/** Mobile viewports need scroll into view before interaction */
90+
requiresScrollIntoView: true,
91+
/** Mobile may have hamburger menu instead of full navigation */
92+
hasResponsiveMenu: true,
93+
/** Mobile viewport may need stabilization after orientation/resize */
94+
needsViewportStabilization: true,
95+
/** Some features are desktop-only (hover states, etc.) */
96+
hasLimitedFeatures: true,
97+
},
98+
chromium: {
99+
/** Chromium is the baseline - no special handling needed */
100+
needsFormValidationWait: false,
101+
hasNSBindingAborted: false,
102+
needsElementStabilityWait: false,
103+
hasDelayedLocalStorage: false,
104+
needsTouchEvents: false,
105+
requiresScrollIntoView: false,
106+
},
107+
} as const;
108+
109+
/**
110+
* Error patterns to filter by browser
111+
*
112+
* These are errors that appear in specific browsers but are not actual failures.
113+
* Use with filterExpectedErrors() in test-utils.ts.
114+
*/
115+
export const BROWSER_EXPECTED_ERRORS = {
116+
firefox: [
117+
'NS_BINDING_ABORTED', // Normal during navigation
118+
'AbortError', // Request abort during navigation
119+
'NetworkError when attempting', // Sometimes appears during fast navigation
120+
],
121+
webkit: [
122+
'Failed to load resource', // Sometimes appears during rapid navigation
123+
'Load request cancelled', // WebKit's equivalent of NS_BINDING_ABORTED
124+
'cancelled', // Generic cancellation error
125+
],
126+
mobile: [
127+
'touch-action', // Touch action warnings
128+
],
129+
chromium: [], // Baseline - no special filtering
130+
all: [
131+
'net::ERR_ABORTED', // Normal when navigation cancels pending requests
132+
'Failed to fetch RSC payload', // Next.js RSC during navigation
133+
],
134+
} as const;
135+
136+
/**
137+
* Mobile device viewport configurations
138+
*
139+
* These match Playwright's device definitions but are exported here for
140+
* custom viewport handling in tests.
141+
*/
142+
export const MOBILE_VIEWPORTS = {
143+
'Mobile Chrome': { width: 393, height: 851, isMobile: true, hasTouch: true },
144+
'Mobile Safari': { width: 390, height: 844, isMobile: true, hasTouch: true },
145+
'Pixel 5': { width: 393, height: 851, isMobile: true, hasTouch: true },
146+
'iPhone 12': { width: 390, height: 844, isMobile: true, hasTouch: true },
147+
'iPhone 13': { width: 390, height: 844, isMobile: true, hasTouch: true },
148+
'Galaxy S21': { width: 360, height: 800, isMobile: true, hasTouch: true },
149+
} as const;
150+
151+
/**
152+
* Browser project names as used in Playwright config
153+
*/
154+
export const BROWSER_PROJECTS = {
155+
CHROMIUM: 'chromium',
156+
FIREFOX: 'firefox',
157+
WEBKIT: 'webkit',
158+
MOBILE_CHROME: 'Mobile Chrome',
159+
MOBILE_SAFARI: 'Mobile Safari',
160+
} as const;
161+
162+
export type BrowserName = 'chromium' | 'firefox' | 'webkit';
163+
export type MobileProjectName = 'Mobile Chrome' | 'Mobile Safari';
164+
export type ProjectName = BrowserName | MobileProjectName;

‎tests/e2e/playwright.config.ts‎

Lines changed: 85 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,22 @@
11
import { defineConfig, devices } from '@playwright/test';
22
import { TEST_DB_PATH, FRONTEND_URL } from './e2e-config';
3+
import { BROWSER_TIMEOUTS } from './browser-config';
34

45
/**
56
* Playwright configuration for E2E tests.
7+
*
8+
* Cross-Browser Compatibility Notes:
9+
* - Chromium: Baseline browser, default timeouts
10+
* - Firefox: +50% timeouts for CSS rendering, filter NS_BINDING_ABORTED errors
11+
* - WebKit: +40% timeouts for element stabilization, retry on flaky tests
12+
* - Mobile: Touch events, scroll handling, extended timeouts for network
13+
*
614
* See https://playwright.dev/docs/test-configuration
715
*/
16+
17+
// Environment variable to filter specific browsers (e.g., E2E_BROWSER_FILTER=chromium,firefox)
18+
const browserFilter = process.env.E2E_BROWSER_FILTER?.split(',').map((b) => b.trim().toLowerCase());
19+
820
export default defineConfig({
921
testDir: './',
1022
testMatch: '*.spec.ts',
@@ -21,7 +33,7 @@ export default defineConfig({
2133
/* Fail the build on CI if you accidentally left test.only in the source code */
2234
forbidOnly: !!process.env.CI,
2335

24-
/* Retry on CI only */
36+
/* Retry on CI only - Firefox and WebKit get extra retries */
2537
retries: process.env.CI ? 2 : 0,
2638

2739
/* SQLite doesn't handle concurrent writes well, so always use 1 worker.
@@ -50,7 +62,7 @@ export default defineConfig({
5062
video: 'retain-on-failure',
5163

5264
/* Maximum time each action such as `click()` can take */
53-
actionTimeout: 10000,
65+
actionTimeout: BROWSER_TIMEOUTS.chromium.action,
5466

5567
/* Don't ignore HTTPS errors - catch certificate issues */
5668
ignoreHTTPSErrors: false,
@@ -61,33 +73,91 @@ export default defineConfig({
6173
}),
6274
},
6375

64-
/* Configure projects for major browsers */
76+
/* Configure projects for major browsers with browser-specific settings */
6577
projects: [
78+
// =========================================================================
79+
// CHROMIUM (Baseline)
80+
// =========================================================================
6681
{
6782
name: 'chromium',
68-
use: { ...devices['Desktop Chrome'] },
83+
use: {
84+
...devices['Desktop Chrome'],
85+
},
6986
},
7087

88+
// =========================================================================
89+
// FIREFOX - Slower CSS rendering, NS_BINDING_ABORTED during navigation
90+
// =========================================================================
7191
{
7292
name: 'firefox',
73-
use: { ...devices['Desktop Firefox'] },
93+
use: {
94+
...devices['Desktop Firefox'],
95+
// Extended timeouts for Firefox's slower CSS rendering
96+
actionTimeout: BROWSER_TIMEOUTS.firefox.action,
97+
// Reduce motion to avoid animation timing issues
98+
contextOptions: {
99+
reducedMotion: 'reduce',
100+
},
101+
},
102+
// Firefox gets extra retries due to timing-sensitive failures
103+
retries: process.env.CI ? 3 : 1,
74104
},
75105

106+
// =========================================================================
107+
// WEBKIT - Delayed element rendering, localStorage timing issues
108+
// =========================================================================
76109
{
77110
name: 'webkit',
78-
use: { ...devices['Desktop Safari'] },
111+
use: {
112+
...devices['Desktop Safari'],
113+
// Extended timeouts for WebKit's delayed rendering
114+
actionTimeout: BROWSER_TIMEOUTS.webkit.action,
115+
},
116+
// WebKit gets extra retries due to element stability issues
117+
retries: process.env.CI ? 3 : 1,
79118
},
80119

81-
/* Test against mobile viewports */
120+
// =========================================================================
121+
// MOBILE CHROME - Touch events, smaller viewports
122+
// =========================================================================
82123
{
83124
name: 'Mobile Chrome',
84-
use: { ...devices['Pixel 5'] },
125+
use: {
126+
...devices['Pixel 5'],
127+
// Ensure touch mode is enabled
128+
isMobile: true,
129+
hasTouch: true,
130+
// Extended timeouts for mobile network and touch handling
131+
actionTimeout: BROWSER_TIMEOUTS.mobile.action,
132+
},
133+
// Mobile gets extra retries due to viewport/touch issues
134+
retries: process.env.CI ? 2 : 1,
85135
},
136+
137+
// =========================================================================
138+
// MOBILE SAFARI - Touch events, WebKit quirks on mobile
139+
// =========================================================================
86140
{
87141
name: 'Mobile Safari',
88-
use: { ...devices['iPhone 12'] },
142+
use: {
143+
...devices['iPhone 12'],
144+
// Ensure touch mode is enabled
145+
isMobile: true,
146+
hasTouch: true,
147+
// Extended timeouts (mobile + WebKit)
148+
actionTimeout: BROWSER_TIMEOUTS.mobile.action,
149+
},
150+
// Mobile Safari gets most retries (mobile + WebKit quirks)
151+
retries: process.env.CI ? 3 : 1,
89152
},
90-
],
153+
].filter((project) => {
154+
// Filter projects if E2E_BROWSER_FILTER is set
155+
if (!browserFilter) return true;
156+
const projectName = project.name.toLowerCase();
157+
return browserFilter.some(
158+
(filter) => projectName.includes(filter) || filter.includes(projectName)
159+
);
160+
}),
91161

92162
/* Run local dev server before starting the tests */
93163
webServer: process.env.CI
@@ -113,8 +183,11 @@ export default defineConfig({
113183
/* Global timeout for each test - increased for CI */
114184
timeout: process.env.CI ? 60000 : 30000,
115185

116-
/* Expect timeout - increased for CI */
186+
/* Expect timeout - increased for CI
187+
* Note: Browser-specific expect timeouts are handled via getBrowserTimeout() in test-utils.ts
188+
* This is the base timeout; Firefox and WebKit tests use 1.5x and 1.4x multipliers respectively
189+
*/
117190
expect: {
118-
timeout: process.env.CI ? 10000 : 5000,
191+
timeout: process.env.CI ? 10000 : BROWSER_TIMEOUTS.chromium.expect,
119192
},
120193
});

0 commit comments

Comments
 (0)