fix flaky tests (#877)

* fix flaky tests

* update

* update

* update

* update

* update

* update

* update

* update

Author: refael-m

.github/workflows/e2e_pr.yml

@@ -1,4 +1,4 @@
-name: Video Player PR
+name: Playwright Tests
 
 on:
   workflow_dispatch:
@@ -33,6 +33,9 @@ jobs:
       - name: Set Deploy Preview URL
         run: echo "PREVIEW_URL=https://deploy-preview-${{ env.PR_NUMBER }}--cld-vp-esm-pages.netlify.app" >> $GITHUB_ENV
 
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps
+
       - name: E2E tests
         run: npm run test:e2e
         env:

test/e2e/components/videoComponent.ts

@@ -23,10 +23,19 @@ export class VideoComponent extends BaseComponent {
      */
     public async isPaused(): Promise<boolean> {
         return this.props.page.evaluate((selector: string) => {
-            console.log('Evaluating selector in browser context:', selector); // Logs selector in browser context
-            const xpathResult = document.evaluate(selector, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
-            const video = xpathResult.singleNodeValue as HTMLVideoElement | null;
-            return video.paused;
+            const video = document.evaluate(
+                selector, 
+                document, 
+                null, 
+                XPathResult.FIRST_ORDERED_NODE_TYPE, 
+                null
+              ).singleNodeValue as HTMLVideoElement | null;
+              
+              if (!video) {
+                throw new Error(`Video element with id "${selector}" not found`);
+              }              
+              return video.paused;
+            
         }, this.props.selector);
     }
 
@@ -37,9 +46,10 @@ export class VideoComponent extends BaseComponent {
      * timeout - Optional. The maximum time (in milliseconds) to wait for the validation. Defaults to 3000ms if not provided.
      * Pass `true` if the video is expected to be playing, or `false` if it is expected to be paused.
      */
-    public async validateVideoIsPlaying(expectedPlaying: boolean, timeout: number = 3000): Promise<void> {
+    public async validateVideoIsPlaying(expectedPlaying: boolean, timeout: number = 6000): Promise<void> {
         await expect(async () => {
-            expect(await this.isPaused()).not.toEqual(expectedPlaying);
+            const isPaused = await this.isPaused();
+            expect(isPaused).not.toEqual(expectedPlaying);
         }).toPass({ intervals: [500], timeout });
     }
 }

test/e2e/playwright.config.ts

@@ -4,7 +4,7 @@ import { defineConfig, devices } from '@playwright/test';
  * See https://playwright.dev/docs/test-configuration.
  */
 export default defineConfig({
-    timeout: 150000,
+    timeout: 60000,
     /* Run tests in files in parallel */
     fullyParallel: true,
     /* Fail the build on CI if you accidentally left test.only in the source code. */
@@ -13,6 +13,8 @@ export default defineConfig({
     retries: process.env.CI ? 2 : 0,
     /* Opt out of parallel tests on CI. */
     workers: 5,
+    /* Max failures to continue the test run */
+    maxFailures: 15,
     /* Reporter to use. See https://playwright.dev/docs/test-reporters */
     reporter: [['list'], ['html', { open: 'never' }]],
     /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
@@ -24,6 +26,7 @@ export default defineConfig({
         /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
         trace: 'retain-on-failure',
         screenshot: 'only-on-failure',
+        video: 'retain-on-failure',
     },
 
     /* Configure projects for major browsers */

test/e2e/specs/NonESM/linksConsolErros.spec.ts

@@ -14,7 +14,9 @@ for (const link of LINKS) {
         await pomPages.mainPage.clickLinkByName(link.name);
         await waitForPageToLoadWithTimeout(page, 5000);
         expect(page.url()).toContain(link.endpoint);
-        handleCommonBrowsersErrors(link.name, consoleErrors);
+        await expect(async () => {
+            handleCommonBrowsersErrors(link.name, consoleErrors);
+          }).toPass({intervals: [1000]});
     });
 }
 

test/e2e/specs/NonESM/visualSearchPage.spec.ts

@@ -18,19 +18,19 @@ vpTest(`Test if video on visual search page is playing as expected`, async ({ pa
         await pomPages.visualSearchPage.visualSearchPlaylistVideoComponent.locator.scrollIntoViewIfNeeded();
     });
 
-    await test.step('Click play button on visual search video', async () => {
+    await test.step('Click play button on visual search plugin video', async () => {
         await pomPages.visualSearchPage.visualSearchVideoComponent.clickPlay();
     });
 
-    await test.step('Validating that visual search video is playing', async () => {
+    await test.step('Validating that visual search plugin video is playing', async () => {
         await pomPages.visualSearchPage.visualSearchVideoComponent.validateVideoIsPlaying(true);
     });
 
-    await test.step('Click play button on playlist video', async () => {
+    await test.step('Click play button on mixed content video', async () => {
         await pomPages.visualSearchPage.visualSearchPlaylistVideoComponent.clickPlay();
     });
 
-    await test.step('Validating that visual search playlist video is playing', async () => {
+    await test.step('Validating that visual mixed content video is playing', async () => {
         await pomPages.visualSearchPage.visualSearchPlaylistVideoComponent.validateVideoIsPlaying(true);
     });
 });

test/e2e/specs/Setup/global.setup.ts

@@ -7,7 +7,8 @@ import { ESM_URL } from '../../testData/esmUrl';
 /**
  * Ensures the esm deploy is ready by checking the 'Adaptive Streaming' link is visible at ESM_URL.
  */
-test('global setup', async ({ page }) => {
+test('Global Setup', async ({ page }) => {
+    test.slow();
     if (ESM_URL) {
         const link: ExampleLinkType = {
             name: ExampleLinkName.AdaptiveStreaming,

test/e2e/specs/commonSpecs/vastAndVpaidPage.ts

@@ -13,9 +13,9 @@ export async function testVastAndVpaidPageVideoIsPlaying(page: Page, pomPages: P
     });
     //Sending timeout of 12 seconds to wait until the ad finishes (10 sec) and the video will start
     await test.step('Validating that single video with ads is playing', async () => {
-        await pomPages.vastAndVpaidPage.singleVideoWithAdsVideoComponent.validateVideoIsPlaying(true, 12000);
+        await pomPages.vastAndVpaidPage.singleVideoWithAdsVideoComponent.validateVideoIsPlaying(true, 20000);
     });
     await test.step('Validating that playlist with ads video is playing', async () => {
-        await pomPages.vastAndVpaidPage.playlistWithAdsVideoComponent.validateVideoIsPlaying(true, 12000);
+        await pomPages.vastAndVpaidPage.playlistWithAdsVideoComponent.validateVideoIsPlaying(true, 20000);
     });
 }

test/e2e/specs/commonSpecs/vr360VideosPageVideoPlaying.ts

@@ -12,6 +12,6 @@ export async function testVr360PageVideoIsPlaying(page: Page, pomPages: PageMana
         return pomPages.vr360VideosPage.vr360VideoComponent.clickPlay();
     });
     await test.step('Validating that 360 video is playing', async () => {
-        await pomPages.vr360VideosPage.vr360VideoComponent.validateVideoIsPlaying(true, 6000);
+        await pomPages.vr360VideosPage.vr360VideoComponent.validateVideoIsPlaying(true, 12000);
     });
 }