Adding visual regression testing infra (#16016)

* Add ability for e2e screenshots

* Change files

* fix

* packages.lock.json

* fix

* Switch away from winrt capture API since it requires user permissions

* fix

* update lock

* publish image snapshots to an artifact

* format

* fix

* lock file

* fix

Author: acoates-ms

.ado/jobs/e2e-test.yml

@@ -74,6 +74,21 @@ jobs:
               workingDirectory: packages/e2e-test-app-fabric
               timeoutInMinutes: 10 # Time to wait for this task to complete before the server kills it.
 
+            - script: |
+                if [ -d "packages/e2e-test-app-fabric/test/__image_snapshots__/__diff_output__" ]; then
+                  echo "##vso[task.setvariable variable=DiffOutputExists]true"
+                fi
+              displayName: Check for image diff output folder
+              condition: failed()
+
+            - task: CopyFiles@2
+              displayName: Copy Fabric image diffs
+              inputs:
+                sourceFolder: packages/e2e-test-app-fabric/test/__image_snapshots__/__diff_output__
+                targetFolder: $(Build.StagingDirectory)/snapshots-image-diffs
+                contents: "**"
+              condition: and(failed(), eq(variables.DiffOutputExists, 'true'))
+
             - script: npx jest --clearCache
               displayName: clear jest cache
               workingDirectory: packages/e2e-test-app-fabric
@@ -103,6 +118,14 @@ jobs:
                 contents: "**"
               condition: failed()
 
+            - task: CopyFiles@2
+              displayName: Copy Fabric image snapshots
+              inputs:
+                sourceFolder: packages/e2e-test-app-fabric/test/__image_snapshots__
+                targetFolder: $(Build.StagingDirectory)/snapshots-image
+                contents: "**"
+              condition: failed()
+
             - task: CopyFiles@2
               displayName: Copy RNTesterApp artifacts
               inputs:
@@ -133,6 +156,16 @@ jobs:
               condition: failed()
               targetPath: $(Build.StagingDirectory)/snapshots-fabric
               artifactName: Snapshots - RNTesterApp-Fabric-${{ matrix.Name }}-$(System.JobAttempt)
+            - output: pipelineArtifact
+              displayName: 'Publish Artifact: Image Diffs'
+              condition: failed()
+              targetPath: $(Build.StagingDirectory)/snapshots-image-diffs
+              artifactName: Snapshots - RNTesterApp-Image-Diffs-${{ matrix.Name }}-$(System.JobAttempt)
+            - output: pipelineArtifact
+              displayName: 'Publish Artifact: Image Snapshots'
+              condition: failed()
+              targetPath: $(Build.StagingDirectory)/snapshots-image
+              artifactName: Snapshots - RNTesterApp-Image-${{ matrix.Name }}-$(System.JobAttempt)
             - output: pipelineArtifact
               displayName: 'Upload build logs'
               condition: succeededOrFailed()

change/@react-native-windows-automation-commands-0c626a05-33dc-4738-b0a2-2414c549c404.json

@@ -0,0 +1,7 @@
+{
+  "type": "prerelease",
+  "comment": "Add ability for e2e screenshots",
+  "packageName": "@react-native-windows/automation-commands",
+  "email": "30809111+acoates-ms@users.noreply.github.com",
+  "dependentChangeType": "patch"
+}

packages/@react-native-windows/automation-commands/src/index.ts

@@ -6,5 +6,6 @@
  */
 
 import dumpVisualTree from './dumpVisualTree';
+import createScreenshot from './screenshot';
 
-export {dumpVisualTree};
+export { dumpVisualTree, createScreenshot };

packages/@react-native-windows/automation-commands/src/screenshot.ts

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT License.
+ *
+ * @format
+ */
+
+/**
+ * Take a screenshot of the app window
+ */
+export default async function createScreenshot(
+  opts?: {
+    accessibilityId: string,
+    screenshotsPath?: string,
+    location?: { x: number, y: number, width: number, height: number };
+  },
+): Promise<void> {
+  if (!automationClient) {
+    throw new Error('RPC client is not enabled');
+  }
+
+  const dumpResponse = await automationClient.invoke('CreateScreenshot', {
+    ...opts,
+  });
+
+  if (dumpResponse.type === 'error') {
+    throw new Error(dumpResponse.message);
+  }
+}

packages/@react-native-windows/tester/src/js/examples/View/ViewExample.windows.js

@@ -424,6 +424,7 @@ function BoxShadowExample(): React.Node {
 
   return (
     <View
+      accessible
       testID="view-test-box-shadow"
       style={{flexDirection: 'row', flexWrap: 'wrap', gap: 30, padding: 20}}>
       <View

packages/e2e-test-app-fabric/jest.setup.js

@@ -12,6 +12,10 @@ const sanitizeFilename = require('sanitize-filename');
 // disabled temporarily
 //const {LogBox} = require('react-native');
 
+const { toMatchImageSnapshot } = require('jest-image-snapshot');
+
+expect.extend({ toMatchImageSnapshot });
+
 const screenshotDir = './errorShots';
 fs.mkdirSync(screenshotDir, {recursive: true});
 

packages/e2e-test-app-fabric/package.json

@@ -43,6 +43,7 @@
     "@react-native-community/cli": "20.0.0",
     "@react-native-windows/automation": "0.0.0-canary.1052",
     "@react-native-windows/automation-commands": "0.0.0-canary.1052",
+    "@react-native-windows/fs": "^0.0.0-canary.72",
     "@react-native-windows/perf-testing": "0.0.0-canary.1038",
     "@react-native/metro-config": "0.85.0-nightly-20260114-f15985f4f",
     "@rnw-scripts/babel-node-config": "2.3.3",
@@ -60,6 +61,7 @@
     "eslint": "^8.19.0",
     "jest": "^29.7.0",
     "jest-environment-node": "^29.7.0",
+    "jest-image-snapshot": "^6.5.2",
     "prettier": "^3.6.2",
     "react-test-renderer": "19.1.0",
     "sanitize-filename": "^1.6.3",

packages/e2e-test-app-fabric/test/Helpers.ts

@@ -5,7 +5,13 @@
  * @format
  */
 
-import {AutomationClient} from '@react-native-windows/automation-channel';
+import type { AutomationElement } from '@react-native-windows/automation';
+import { AutomationClient } from '@react-native-windows/automation-channel';
+
+import { createScreenshot } from '@react-native-windows/automation-commands';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import fs from '@react-native-windows/fs';
 
 declare global {
   const automationClient: AutomationClient | undefined;
@@ -15,6 +21,23 @@ type ErrorsResult = {
   errors: string[];
 };
 
+export interface ImageSnapshotConfig {
+  failureThreshold?: number;
+  failureThresholdType?: 'percent'
+}
+
+export async function verifyElementVisualSnapshot(component: AutomationElement, config?: ImageSnapshotConfig) {
+  const { x, y } = await component.getLocation();
+  const { width, height } = await component.getSize();
+  await createScreenshot({ screenshotsPath: tmpdir(), location: { x, y, width, height } });
+  const myFancyImage = fs.readFileSync(join(tmpdir(), "./RNTester.png"));
+  expect(myFancyImage).toMatchImageSnapshot({
+    failureThreshold: 0.01,
+    failureThresholdType: 'percent',
+    ...config
+  });
+}
+
 export async function verifyNoErrorLogs(
   errorFilter?: (errors: string[]) => string[],
 ): Promise<void> {

packages/e2e-test-app-fabric/test/ViewComponentTest.test.ts

@@ -5,10 +5,10 @@
  * @format
  */
 
-import {dumpVisualTree} from '@react-native-windows/automation-commands';
-import {goToComponentExample} from './RNTesterNavigation';
-import {app} from '@react-native-windows/automation';
-import {verifyNoErrorLogs} from './Helpers';
+import { dumpVisualTree } from '@react-native-windows/automation-commands';
+import { goToComponentExample } from './RNTesterNavigation';
+import { app } from '@react-native-windows/automation';
+import { verifyNoErrorLogs, verifyElementVisualSnapshot } from './Helpers';
 
 beforeAll(async () => {
   // If window is partially offscreen, tests will fail to click on certain elements
@@ -40,14 +40,23 @@ describe('View Tests', () => {
   test('Views can have shadows', async () => {
     await searchBox('sha');
     const component = await app.findElementByTestID('shadow');
-    await component.waitForDisplayed({timeout: 5000});
+    await component.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('shadow');
     expect(dump).toMatchSnapshot();
   });
+  test('Visual Snapshot of Views with shadows', async () => {
+    await searchBox('box shadow');
+    const component = await app.findElementByTestID('view-test-box-shadow');
+    await component.waitForDisplayed({ timeout: 5000 });
+    const dump = await dumpVisualTree('view-test-box-shadow');
+    expect(dump).toMatchSnapshot();
+
+    await verifyElementVisualSnapshot(component);
+  })
   test('Views can have border styles', async () => {
     await searchBox('sty');
     const componentsTab = await app.findElementByTestID('border-style-button');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('border-style-button');
     expect(dump).toMatchSnapshot();
   });
@@ -56,21 +65,21 @@ describe('View Tests', () => {
     const componentsTab = await app.findElementByTestID(
       'offscreen-alpha-compositing-button',
     );
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('offscreen-alpha-compositing-button');
     expect(dump).toMatchSnapshot();
   });
   test('Views can have a z-index', async () => {
     await searchBox('z');
     const componentsTab = await app.findElementByTestID('z-index-button');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('z-index-button');
     expect(dump).toMatchSnapshot();
   });
   test('Views can have display: none', async () => {
     await searchBox('dis');
     const componentsTab = await app.findElementByTestID('display-none-button');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('display-none-button');
     expect(dump).toMatchSnapshot();
   });
@@ -80,14 +89,14 @@ describe('View Tests', () => {
     const componentsTab = await app.findElementByTestID(
       'view-test-background-color',
     );
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('view-test-background-color');
     expect(dump).toMatchSnapshot();
   });
   test('Views can have borders', async () => {
     await searchBox('bor');
     const componentsTab = await app.findElementByTestID('view-test-border');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('view-test-border');
     expect(dump).toMatchSnapshot();
   });
@@ -96,7 +105,7 @@ describe('View Tests', () => {
     const componentsTab = await app.findElementByTestID(
       'view-test-padding-margin',
     );
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('view-test-padding-margin');
     expect(dump).toMatchSnapshot();
   });
@@ -105,14 +114,14 @@ describe('View Tests', () => {
     const componentsTab = await app.findElementByTestID(
       'view-test-rounded-borders',
     );
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('view-test-rounded-borders');
     expect(dump).toMatchSnapshot();
   });
   test('Views can have overflow', async () => {
     await searchBox('ove');
     const componentsTab = await app.findElementByTestID('view-test-overflow');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('view-test-overflow');
     expect(dump).toMatchSnapshot();
   });
@@ -121,21 +130,21 @@ describe('View Tests', () => {
     const componentsTab = await app.findElementByTestID(
       'view-test-rounded-borders',
     );
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('view-test-rounded-borders');
     expect(dump).toMatchSnapshot();
   });
   test('Views can have customized opacity', async () => {
     await searchBox('opa');
     const componentsTab = await app.findElementByTestID('view-test-opacity');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('view-test-opacity');
     expect(dump).toMatchSnapshot();
   });
   test('Views can have tooltips', async () => {
     await searchBox('too');
     const componentsTab = await app.findElementByTestID('tool-tip');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('tool-tip');
     expect(dump).toMatchSnapshot();
   });
@@ -144,42 +153,42 @@ describe('View Tests', () => {
     const componentsTab = await app.findElementByTestID(
       'view-test-backface-visibility',
     );
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('view-test-backface-visibility');
     expect(dump).toMatchSnapshot();
   });
   test('Views can have aria-labels', async () => {
     await searchBox('ari');
     const componentsTab = await app.findElementByTestID('view-test-aria-label');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('view-test-aria-label');
     expect(dump).toMatchSnapshot();
   });
   test('Views can have flexgap', async () => {
     await searchBox('fle');
     const componentsTab = await app.findElementByTestID('view-test-flexgap');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('view-test-flexgap');
     expect(dump).toMatchSnapshot();
   });
   test('Views can have insets', async () => {
     await searchBox('ins');
     const componentsTab = await app.findElementByTestID('view-test-insets');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('view-test-insets');
     expect(dump).toMatchSnapshot();
   });
   test('Views can have customized accessibility', async () => {
     await searchBox('acc');
     const componentsTab = await app.findElementByTestID('accessibility');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('accessibility');
     expect(dump).toMatchSnapshot();
   });
   test('Views can have a hitslop region', async () => {
     await searchBox('hit');
     const componentsTab = await app.findElementByTestID('hitslop');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('hitslop');
     expect(dump).toMatchSnapshot();
   });
@@ -188,14 +197,14 @@ describe('View Tests', () => {
     const componentsTab = await app.findElementByTestID(
       'view-test-logical-border-color',
     );
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('view-test-logical-border-color');
     expect(dump).toMatchSnapshot();
   });
   test('Views can have a nativeid', async () => {
     await searchBox('nat');
     const componentsTab = await app.findElementByTestID('nativeid');
-    await componentsTab.waitForDisplayed({timeout: 5000});
+    await componentsTab.waitForDisplayed({ timeout: 5000 });
     const dump = await dumpVisualTree('nativeid');
     expect(dump).toMatchSnapshot();
   });