Fix flaky e2e tests (#951)

* Fix

* Change discovery test to serial

* Add global-setup.ts

* Change playwright trace config

* Update comment

* CR suggestions

* Change number of workers

Author: hhubert6

e2e/global-setup.ts

@@ -0,0 +1,35 @@
+/**
+ * Warm-up that verifies the tracing infrastructure is fully initialized.
+ *
+ * TracingManager.handle_continue(:setup_tracing) runs asynchronously after app boot.
+ * We open a dev app + debugger pair and check if traces were captured.
+ * If not, we trigger callback in dev app and check for traces with timeout (repeated 5 times).
+ */
+import { chromium } from '@playwright/test';
+
+export default async function globalSetup() {
+  const browser = await chromium.launch();
+  const page = await browser.newPage();
+
+  await page.goto(
+    process.env.PLAYWRIGHT_TEST_BASE_URL ?? 'http://localhost:4005/'
+  );
+
+  await page.locator('#live-debugger-debug-button').click();
+  const dbgAppPromise = page.waitForEvent('popup');
+  await page.getByText('Open in new tab').click();
+  const dbgApp = await dbgAppPromise;
+
+  for (let i = 0; i < 5; i++) {
+    try {
+      await dbgApp
+        .locator('#traces-list-stream details')
+        .waitFor({ timeout: 2000 });
+      break;
+    } catch {
+      await page.locator('#increment-button').click();
+    }
+  }
+
+  await browser.close();
+}

e2e/playwright.config.ts

@@ -4,14 +4,15 @@ import { defineConfig, devices } from '@playwright/test';
  * See https://playwright.dev/docs/test-configuration.
  */
 export default defineConfig({
+  globalSetup: './global-setup.ts',
   testDir: './tests',
   /* Run tests in files in parallel */
   fullyParallel: true,
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
   /* Retry on CI only */
   retries: process.env.CI ? 2 : 0,
-  workers: process.env.CI ? 2 : undefined,
+  workers: process.env.CI ? 4 : undefined,
   maxFailures: process.env.CI ? 5 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: 'html',
@@ -21,7 +22,7 @@ export default defineConfig({
     baseURL: 'http://localhost:4005',
 
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
-    trace: 'on-first-retry',
+    trace: 'retain-on-failure',
   },
 
   projects: [
@@ -38,11 +39,13 @@ export default defineConfig({
     {
       name: 'serial-tests chromium',
       use: { ...devices['Desktop Chrome'] },
+      workers: 1,
       testMatch: '**/*.serial.spec.ts',
       dependencies: ['chromium', 'firefox'],
     },
     {
       name: 'serial-tests firefox',
+      workers: 1,
       use: { ...devices['Desktop Firefox'] },
       testMatch: '**/*.serial.spec.ts',
       dependencies: ['chromium', 'firefox', 'serial-tests chromium'],

e2e/tests/assigns.spec.ts

@@ -28,13 +28,13 @@ const showButton = async (page: Page, selector: string) => {
 const clickPinButton = async (page: Page, assignKey: string) => {
   const selector = `#all-assigns button[phx-click="pin-assign"][phx-value-key="${assignKey}"]`;
   await showButton(page, selector);
-  await page.locator(selector).click();
+  await page.locator(selector).click({ force: true });
 };
 
 const clickUnpinButton = async (page: Page, assignKey: string) => {
   const selector = `#pinned-assigns button[phx-click="unpin-assign"][phx-value-key="${assignKey}"]`;
   await showButton(page, selector);
-  await page.locator(selector).click();
+  await page.locator(selector).click({ force: true });
 };
 
 test('user can search assigns using the searchbar', async ({ dbgApp }) => {

e2e/tests/discovery.spec.ts e2e/tests/discovery.serial.spec.tse2e/tests/discovery.spec.ts renamed to e2e/tests/discovery.serial.spec.ts

@@ -25,7 +25,7 @@ test('user can see active live views and their highlights which are refreshed au
   await expect(dbgApp.getByText(devPidString1)).toBeVisible();
 
   const devApp2 = await context.newPage();
-  devApp2.goto('/');
+  await devApp2.goto('/');
 
   const devPidString2 =
     (await devApp2.locator('#current-pid').textContent()) ?? '';
@@ -62,7 +62,7 @@ test('settings button exists and redirects works as expected', async ({
 
   await dbgApp.getByRole('link', { name: 'Icon settings' }).click();
 
-  await expect(dbgApp.getByRole('heading')).toHaveText('Settings');
+  await expect(dbgApp.getByRole('heading', { name: 'Settings' })).toBeVisible();
 
   await dbgApp.getByRole('link', { name: 'Icon arrow left' }).click();
 

e2e/tests/node-inspector.spec.ts

@@ -74,7 +74,7 @@ test('callback traces have proper execution times displayed', async ({
 
   await expect(traces).toHaveCount(2);
   await expect(traces.last().locator('span.text-warning-text')).toHaveText(
-    /40\d ms/
+    /^\s*4\d\d ms\s*$/
   );
 
   await devApp
@@ -87,7 +87,7 @@ test('callback traces have proper execution times displayed', async ({
 
   await expect(traces).toHaveCount(4);
   await expect(traces.nth(1).locator('span.text-error-text')).toHaveText(
-    /1\.10 s/
+    /^\s*1\.1\d s\s*$/
   );
 });