fix: improve E2E test stability and reduce flakiness (calcom#25897)

* fix: improve E2E test stability and reduce flakiness

- Replace hardcoded waits with proper Playwright waitFor assertions in slot selection
- Add explicit waits for calendar and time slot elements to be visible before clicking
- Replace fixed 2s wait in gotoRoutingLink with networkidle wait
- Replace fixed 5s email wait with retry logic (10 retries, 500ms intervals)
- Use unique usernames with timestamps to avoid parallel test collisions
- Use features fixture instead of direct prisma calls for feature flag mutations
- Fix login.e2e.ts to use unique username instead of hardcoded 'pro'
- Fix signup.e2e.ts to use unique usernames and proper feature flag handling
- Remove unused 'users' parameter from tests that don't need it

Co-Authored-By: Volnei Munhoz <volnei.munhoz@gmail.com>

* perf: replace waitForTimeout with smart waits for faster E2E tests

- Replace 30 waitForTimeout calls with proper Playwright waits
- Use waitFor({ state: 'visible' }) for element visibility
- Use waitForLoadState('networkidle') for page load completion
- Use waitForFunction for localStorage state changes

Performance improvements:
- limit-tab.e2e.ts: 10s fixed wait -> element visibility wait
- booking-duplicate-api-calls.e2e.ts: 5s fixed wait -> networkidle
- change-theme.e2e.ts: 3s fixed wait -> localStorage state check
- team-invitation.e2e.ts: multiple 500ms-3s waits -> element waits
- booking-seats.e2e.ts: 2s waits -> dropdown visibility waits
- embed-code-generator.e2e.ts: 1s waits -> iframe visibility waits
- organization-privacy.e2e.ts: 500ms waits -> element/networkidle waits
- organization-invitation.e2e.ts: 500ms-1s waits -> element waits
- analyticsApps.e2e.ts: 1s wait -> networkidle
- integrations.e2e.ts: 1s wait -> calendar element wait
- fixtures/apps.ts: 1s waits -> element visibility waits

Co-Authored-By: Volnei Munhoz <volnei.munhoz@gmail.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

Author: volnei

apps/web/playwright/apps/analytics/analyticsApps.e2e.ts

@@ -17,8 +17,8 @@ test.describe("check analytics Apps", () => {
         await user.apiLogin();
         await page.goto("apps/categories/analytics");
         await appsPage.installAnalyticsAppSkipConfigure(app);
-        // eslint-disable-next-line playwright/no-wait-for-timeout
-        await page.waitForTimeout(1000); // waits for 1 second
+        // Wait for navigation to complete instead of fixed 1s wait
+        await page.waitForLoadState("networkidle");
         await page.goto("/event-types");
         await appsPage.goToEventType("30 min");
         await appsPage.goToAppsTab();

apps/web/playwright/booking-duplicate-api-calls.e2e.ts

@@ -27,7 +27,12 @@ async function testDuplicateAPICalls(
   });
 
   await page.goto(url);
-  await page.waitForTimeout(5000);
+  // Wait for the booking page to fully load and time slots to appear instead of fixed 5s wait
+  await page.waitForLoadState("networkidle");
+  // Also wait for the calendar to be visible as a more specific indicator
+  await page.locator('[data-testid="calendar"]').or(page.locator('[data-testid="time"]')).first().waitFor({ state: "visible", timeout: 10000 }).catch(() => {
+    // If neither element appears, the page might have a different structure - continue anyway
+  });
 
   return {
     totalCalls: trpcCalls.length + apiV2Calls.length,

apps/web/playwright/booking-seats.e2e.ts

@@ -555,15 +555,17 @@ test.describe("Reschedule for booking with seats", () => {
     await page.waitForSelector('[data-testid="bookings"]');
 
     await page.locator('[data-testid="booking-actions-dropdown"]').nth(0).click();
-    await page.waitForTimeout(2000);
+    // Wait for the dropdown menu to appear instead of fixed 2s wait
+    await page.locator('[data-testid="reschedule"]').waitFor({ state: "visible" });
     const href = await page.locator('[data-testid="reschedule"]').getAttribute("href");
     const url = new URL(href!, page.url());
     const seatReferenceUid = url.searchParams.get('seatReferenceUid');
     if(!seatReferenceUid) {
       await page.reload();
       await page.waitForSelector('[data-testid="bookings"]');
       await page.locator('[data-testid="booking-actions-dropdown"]').nth(0).click();
-      await page.waitForTimeout(2000);
+      // Wait for the dropdown menu to appear instead of fixed 2s wait
+      await page.locator('[data-testid="reschedule"]').waitFor({ state: "visible" });
     }
     await page.locator('[data-testid="reschedule"]').click();
     await expect(page.getByText("Seats available").first()).toBeVisible();

apps/web/playwright/change-theme.e2e.ts

@@ -55,7 +55,8 @@ test.describe("Change App Theme Test", () => {
     const toast2 = await page.waitForSelector('[data-testid="toast-success"]');
     expect(toast2).toBeTruthy();
 
-    await page.waitForTimeout(3000);
+    // Wait for localStorage to be updated after theme change instead of fixed 3s wait
+    await page.waitForFunction(() => localStorage.getItem("app-theme") !== null, { timeout: 5000 });
     const themeValue = await page.evaluate(() => localStorage.getItem("app-theme"));
     expect(themeValue).toBe("light");
 

apps/web/playwright/embed-code-generator.e2e.ts

@@ -59,8 +59,8 @@ test.describe("Embed Code Generator Tests", () => {
           orgSlug: null,
         });
 
-        // To prevent early timeouts
-        await page.waitForTimeout(1000);
+        // Wait for the preview iframe to be ready instead of fixed 1s wait
+        await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
         await expectToContainValidPreviewIframe(page, {
           embedType: "inline",
           calLink: `${pro.username}/multiple-duration`,
@@ -96,8 +96,8 @@ test.describe("Embed Code Generator Tests", () => {
           orgSlug: null,
         });
 
-        // To prevent early timeouts
-        await page.waitForTimeout(1000);
+        // Wait for the preview iframe to be ready instead of fixed 1s wait
+        await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
         await expectToContainValidPreviewIframe(page, {
           embedType: "floating-popup",
           calLink: `${pro.username}/multiple-duration`,
@@ -133,8 +133,8 @@ test.describe("Embed Code Generator Tests", () => {
           orgSlug: null,
         });
 
-        // To prevent early timeouts
-        await page.waitForTimeout(1000);
+        // Wait for the preview iframe to be ready instead of fixed 1s wait
+        await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
         await expectToContainValidPreviewIframe(page, {
           embedType: "element-click",
           calLink: `${pro.username}/multiple-duration`,
@@ -172,8 +172,8 @@ test.describe("Embed Code Generator Tests", () => {
           orgSlug: null,
         });
 
-        // To prevent early timeouts
-        await page.waitForTimeout(1000);
+        // Wait for the preview iframe to be ready instead of fixed 1s wait
+        await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
         await expectToContainValidPreviewIframe(page, {
           embedType: "inline",
           calLink: decodeURIComponent(embedUrl),
@@ -229,8 +229,8 @@ test.describe("Embed Code Generator Tests", () => {
           orgSlug: org.slug,
         });
 
-        // To prevent early timeouts
-        await page.waitForTimeout(1000);
+        // Wait for the preview iframe to be ready instead of fixed 1s wait
+        await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
         await expectToContainValidPreviewIframe(page, {
           embedType: "inline",
           calLink: `${user.username}/multiple-duration`,
@@ -269,8 +269,8 @@ test.describe("Embed Code Generator Tests", () => {
           orgSlug: org.slug,
         });
 
-        // To prevent early timeouts
-        await page.waitForTimeout(1000);
+        // Wait for the preview iframe to be ready instead of fixed 1s wait
+        await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
         await expectToContainValidPreviewIframe(page, {
           embedType: "floating-popup",
           calLink: `${user.username}/multiple-duration`,
@@ -308,8 +308,8 @@ test.describe("Embed Code Generator Tests", () => {
           orgSlug: org.slug,
         });
 
-        // To prevent early timeouts
-        await page.waitForTimeout(1000);
+        // Wait for the preview iframe to be ready instead of fixed 1s wait
+        await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
         await expectToContainValidPreviewIframe(page, {
           embedType: "element-click",
           calLink: `${user.username}/multiple-duration`,
@@ -326,8 +326,8 @@ function chooseEmbedType(page: Page, embedType: EmbedType) {
 }
 
 async function goToReactCodeTab(page: Page) {
-  // To prevent early timeo
-  await page.waitForTimeout(1000);
+  // Wait for the React tab to be visible instead of fixed 1s wait
+  await page.locator("[data-testid=horizontal-tab-react]").waitFor({ state: "visible" });
   await page.locator("[data-testid=horizontal-tab-react]").click();
 }
 

apps/web/playwright/eventType/limit-tab.e2e.ts

@@ -18,7 +18,8 @@ test.describe("Limits Tab - Event Type", () => {
     await bookingPage.updateEventType();
     const eventTypePage = await bookingPage.previewEventType();
 
-    await eventTypePage.waitForTimeout(10000);
+    // Wait for time slots to load instead of fixed 10s wait
+    await eventTypePage.getByTestId("time").first().waitFor({ state: "visible", timeout: 30000 });
 
     const counter = await eventTypePage.getByTestId("time").count();
     await bookingPage.checkTimeSlotsCount(eventTypePage, counter);

apps/web/playwright/fixtures/apps.ts

@@ -33,8 +33,8 @@ export function createAppsFixture(page: Page) {
       await page.click('[data-testid="install-app-button-personal"]');
       await page.waitForURL(`apps/installation/event-types?slug=${app}`);
 
-      // eslint-disable-next-line playwright/no-wait-for-timeout
-      await page.waitForTimeout(1000);
+      // Wait for event type checkboxes to be visible instead of fixed 1s wait
+      await page.locator('[data-testid^="select-event-type-"]').first().waitFor({ state: "visible" });
       for (const id of eventTypeIds) {
         await page.click(`[data-testid="select-event-type-${id}"]`);
       }
@@ -81,8 +81,8 @@ export function createAppsFixture(page: Page) {
       await page.getByTestId("install-app-button").click();
       await page.waitForURL(`apps/installation/event-types?slug=${app.slug}`);
 
-      // eslint-disable-next-line playwright/no-wait-for-timeout
-      await page.waitForTimeout(1000);
+      // Wait for event type checkboxes to be visible instead of fixed 1s wait
+      await page.locator('[data-testid^="select-event-type-"]').first().waitFor({ state: "visible" });
       for (const id of eventTypeIds) {
         await page.click(`[data-testid="select-event-type-${id}"]`);
       }

apps/web/playwright/integrations.e2e.ts

@@ -99,13 +99,10 @@ const addLocationIntegrationToFirstEvent = async function ({ user }: { user: { u
 };
 
 async function bookEvent(page: Page, calLink: string) {
-  // Let current month dates fully render.
-  // There is a bug where if we don't let current month fully render and quickly click go to next month, current month gets rendered
-  // This doesn't seem to be replicable with the speed of a person, only during automation.
-  // It would also allow correct snapshot to be taken for current month.
-  // eslint-disable-next-line playwright/no-wait-for-timeout
-  await page.waitForTimeout(1000);
+  // Navigate to the booking page and wait for calendar to be ready
   await page.goto(`/${calLink}`);
+  // Wait for the calendar day elements to be visible instead of fixed 1s wait
+  await page.locator('[data-testid="day"][data-disabled="false"]').first().waitFor({ state: "visible" });
 
   await page.locator('[data-testid="day"][data-disabled="false"]').nth(0).click();
   page.locator('[data-testid="time"]').nth(0).click();

apps/web/playwright/lib/testUtils.ts

@@ -109,19 +109,30 @@ export async function selectFirstAvailableTimeSlotNextMonth(page: Page | Frame)
   // Let current month dates fully render.
   await page.getByTestId("incrementMonth").click();
 
-  // Waiting for full month increment
-  await page.locator('[data-testid="day"][data-disabled="false"]').nth(0).click();
-
-  await page.locator('[data-testid="time"]').nth(0).click();
+  // Wait for the calendar to update after month increment before clicking
+  const availableDay = page.locator('[data-testid="day"][data-disabled="false"]').nth(0);
+  await availableDay.waitFor({ state: "visible" });
+  await availableDay.click();
+
+  // Wait for time slots to load after selecting a day
+  const timeSlot = page.locator('[data-testid="time"]').nth(0);
+  await timeSlot.waitFor({ state: "visible" });
+  await timeSlot.click();
 }
 
 export async function selectSecondAvailableTimeSlotNextMonth(page: Page) {
   // Let current month dates fully render.
   await page.getByTestId("incrementMonth").click();
 
-  await page.locator('[data-testid="day"][data-disabled="false"]').nth(1).click();
+  // Wait for the calendar to update after month increment before clicking
+  const availableDay = page.locator('[data-testid="day"][data-disabled="false"]').nth(1);
+  await availableDay.waitFor({ state: "visible" });
+  await availableDay.click();
 
-  await page.locator('[data-testid="time"]').nth(0).click();
+  // Wait for time slots to load after selecting a day
+  const timeSlot = page.locator('[data-testid="time"]').nth(0);
+  await timeSlot.waitFor({ state: "visible" });
+  await timeSlot.click();
 }
 
 export async function bookEventOnThisPage(page: Page) {
@@ -253,8 +264,9 @@ export async function gotoRoutingLink({
 
   await page.goto(`${previewLink}${queryString ? `?${queryString}` : ""}`);
 
-  // HACK: There seems to be some issue with the inputs to the form getting reset if we don't wait.
-  await new Promise((resolve) => setTimeout(resolve, 2000));
+  // Wait for the form to be fully loaded and stable before interacting
+  // This replaces the previous hardcoded 2s wait with a proper Playwright wait
+  await page.waitForLoadState("networkidle");
 }
 
 export async function installAppleCalendar(page: Page) {
@@ -274,21 +286,31 @@ export async function getInviteLink(page: Page) {
 export async function getEmailsReceivedByUser({
   emails,
   userEmail,
-  waitForEmailMs = 5000,
+  maxRetries = 10,
+  retryIntervalMs = 500,
 }: {
   emails?: ReturnType<typeof createEmailsFixture>;
   userEmail: string;
-  waitForEmailMs?: number;
+  maxRetries?: number;
+  retryIntervalMs?: number;
 }): Promise<Messages | null> {
   if (!emails) return null;
 
-  // Wait for email to be sent/received
-  await new Promise((resolve) => setTimeout(resolve, waitForEmailMs));
+  // Use retry logic instead of a fixed wait to handle email delivery timing
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    const matchingEmails = await emails.search(userEmail, "to");
+    if (matchingEmails?.total) {
+      return matchingEmails;
+    }
+    // Wait before retrying
+    await new Promise((resolve) => setTimeout(resolve, retryIntervalMs));
+  }
 
+  // Final attempt after all retries
   const matchingEmails = await emails.search(userEmail, "to");
   if (!matchingEmails?.total) {
     console.log(
-      `No emails received by ${userEmail}. All emails sent to:`,
+      `No emails received by ${userEmail} after ${maxRetries} retries. All emails sent to:`,
       (await emails.messages())?.items.map((e) => e.to)
     );
   }

apps/web/playwright/login.e2e.ts

@@ -72,10 +72,11 @@ test.describe("Login and logout tests", () => {
     test("Should warn when password is incorrect", async ({ page, users }) => {
       const alertMessage = (await localize("en"))("incorrect_email_password");
       // by default password===username with the users fixture
-      const pro = await users.create({ username: "pro" });
+      // Use unique username to avoid collisions with parallel tests
+      const user = await users.create();
 
       // login with a wrong password
-      await login({ username: pro.username, password: "wrong" }, page);
+      await login({ username: user.username, password: "wrong" }, page);
 
       // assert for the visibility of the localized  alert message
       await expect(page.locator(`text=${alertMessage}`)).toBeVisible();