cameraCapture: Simplify codepaths, try to wait for a real frame (#4365)

- Don't use fallback paths in camera capture; instead, have the test pick the exact path it's using, and parameterize over it. Test two paths: VideoFrame from MediaStreamTrackProcessor, and the old-fashioned HTMLVideoElement.
- Work around an issue where Chrome (at least on Mac) shows blank frames for a while after initializing the camera.
- I happened to notice that requesting different width/height from the camera is broken in Chrome in several ways, so added cases for that.
- Added a TODO for copyExternalImageToTexture from camera

These tests only worked in Chrome, and even then they were buggy (usually running the test on a blank first frame). Now, the HTMLVideoElement tests work in Safari.

Test failures are as follows, on an M1 Mac:
- Chrome: https://crbug.com/411656657
  - HTMLVideoElement passes some cases, but fails when a requested size is passed to getUserMedia().
  - VideoFrame fails due to incorrect color management (regardless of dstColorSpace).
- Safari:
  - HTMLVideoElement fails due to incorrect color management (regardless of dstColorSpace).
  - VideoFrame is skipped.
- Firefox: doesn't yet implement importExternalTexture.

Issue: fixes #4363, cc #4364

Author: kainino0x

src/webgpu/web_platform/external_texture/video.spec.ts

@@ -6,9 +6,11 @@ Tests for external textures from HTMLVideoElement (and other video-type sources?
 
 TODO: consider whether external_texture and copyToTexture video tests should be in the same file
 TODO(#3193): Test video in BT.2020 color space
+TODO(#4364): Test camera capture with copyExternalImageToTexture (not necessarily in this file)
 `;
 
 import { makeTestGroup } from '../../../common/framework/test_group.js';
+import { unreachable } from '../../../common/util/util.js';
 import { GPUTest } from '../../gpu_test.js';
 import * as ttu from '../../texture_test_utils.js';
 import { TextureUploadingUtils } from '../../util/copy_to_texture.js';
@@ -17,12 +19,13 @@ import {
   startPlayingAndWaitForVideo,
   getVideoFrameFromVideoElement,
   getVideoElement,
-  captureCameraFrame,
   convertToUnorm8,
   kPredefinedColorSpace,
   kVideoNames,
   kVideoInfo,
   kVideoExpectedColors,
+  getVideoElementFromCamera,
+  getVideoFrameFromCamera,
 } from '../../web_platform/util.js';
 
 const kHeight = 16;
@@ -633,34 +636,64 @@ compared with 2d canvas rendering result.
 `
   )
   .params(u =>
+    // NOTE: Be careful not to add too many parameters here, this test is SLOW to initialize!
     u //
       .combineWithParams(checkNonStandardIsZeroCopyIfAvailable())
+      .combine('path', ['HTMLVideoElement', 'MediaStreamTrackProcessor'] as const)
       .combine('dstColorSpace', kPredefinedColorSpace)
+      .combine('constraints', [
+        true,
+        { width: 64, height: 48 },
+        { width: 100, height: 300 },
+      ] as const)
   )
   .fn(async t => {
-    const { dstColorSpace } = t.params;
-
-    const frame = await captureCameraFrame(t);
-
-    if (frame.displayHeight === 0 || frame.displayWidth === 0) {
-      t.skip('Captured video frame has 0 height or width.');
+    const { path, dstColorSpace, constraints } = t.params;
+
+    // Enable this while debugging to show the "actual" and "expected" canvases on screen.
+    const kDebugShowCanvasesOnScreen = false;
+
+    let source: HTMLVideoElement | VideoFrame;
+    let frameWidth: number, frameHeight: number;
+    switch (path) {
+      case 'HTMLVideoElement': {
+        const video = await getVideoElementFromCamera(t, constraints, true);
+        frameWidth = video.videoWidth;
+        frameHeight = video.videoHeight;
+        source = video;
+        break;
+      }
+      case 'MediaStreamTrackProcessor': {
+        const frame = await getVideoFrameFromCamera(t, constraints);
+        frameWidth = frame.displayWidth;
+        frameHeight = frame.displayHeight;
+        source = frame;
+        break;
+      }
+      default:
+        unreachable();
     }
 
-    const frameWidth = frame.displayWidth;
-    const frameHeight = frame.displayHeight;
-
     // Use WebGPU + GPUExternalTexture to render the captured frame.
-    const colorAttachment = t.createTextureTracked({
+    const webgpuCanvas = createCanvas(t, 'onscreen', frameWidth, frameHeight);
+    if (kDebugShowCanvasesOnScreen) {
+      document.body.append(document.createElement('br'));
+      document.body.append(webgpuCanvas);
+    }
+
+    const webgpuContext = webgpuCanvas.getContext('webgpu')!;
+    webgpuContext.configure({
+      device: t.device,
       format: kFormat,
-      size: { width: frameWidth, height: frameHeight },
-      usage: GPUTextureUsage.COPY_SRC | GPUTextureUsage.RENDER_ATTACHMENT,
+      usage: GPUTextureUsage.RENDER_ATTACHMENT | GPUTextureUsage.COPY_SRC,
     });
+    const colorAttachment = webgpuContext.getCurrentTexture();
 
     const pipeline = createExternalTextureSamplingTestPipeline(t);
     const bindGroup = createExternalTextureSamplingTestBindGroup(
       t,
       t.params.checkNonStandardIsZeroCopy,
-      frame,
+      source,
       pipeline,
       dstColorSpace
     );
@@ -684,6 +717,9 @@ compared with 2d canvas rendering result.
 
     // Use 2d context canvas as expected result.
     const canvas = createCanvas(t, 'onscreen', frameWidth, frameHeight);
+    if (kDebugShowCanvasesOnScreen) {
+      document.body.append(canvas);
+    }
 
     const canvasContext = canvas.getContext('2d', { colorSpace: dstColorSpace });
 
@@ -692,7 +728,7 @@ compared with 2d canvas rendering result.
     }
 
     const ctx = canvasContext as CanvasRenderingContext2D;
-    ctx.drawImage(frame, 0, 0, frameWidth, frameHeight);
+    ctx.drawImage(source, 0, 0, frameWidth, frameHeight);
 
     const imageData = ctx.getImageData(0, 0, frameWidth, frameHeight, {
       colorSpace: dstColorSpace,

src/webgpu/web_platform/util.ts

@@ -2,7 +2,7 @@ import { Fixture, SkipTestCase } from '../../common/framework/fixture.js';
 import { getResourcePath } from '../../common/framework/resources.js';
 import { keysOf } from '../../common/util/data_tables.js';
 import { timeout } from '../../common/util/timeout.js';
-import { ErrorWithExtra, raceWithRejectOnTimeout } from '../../common/util/util.js';
+import { ErrorWithExtra, assert, raceWithRejectOnTimeout } from '../../common/util/util.js';
 import { GPUTest } from '../gpu_test.js';
 import { RGBA, srgbToDisplayP3 } from '../util/color_space_conversion.js';
 
@@ -559,59 +559,145 @@ function callbackHelper(
 }
 
 /**
- * Create VideoFrame from camera captured frame. Check whether browser environment has
- * camera supported.
- * Returns a webcodec VideoFrame.
- *
- * @param test: GPUTest that requires getting VideoFrame
- *
+ * Get a MediaStream from the default webcam via `getUserMedia()`.
  */
-export async function captureCameraFrame(test: GPUTest): Promise<VideoFrame> {
+async function getStreamFromCamera(
+  test: Fixture,
+  videoTrackConstraints: MediaTrackConstraints | true
+): Promise<MediaStream> {
   test.skipIf(typeof navigator === 'undefined', 'navigator does not exist in this environment');
   test.skipIf(
     typeof navigator.mediaDevices === 'undefined' ||
       typeof navigator.mediaDevices.getUserMedia === 'undefined',
     "Browser doesn't support capture frame from camera."
   );
 
-  const stream = await navigator.mediaDevices.getUserMedia({ video: true });
-  const track = stream.getVideoTracks()[0] as MediaStreamVideoTrack;
-
-  test.skipIf(!track, "Doesn't have valid camera captured stream for testing.");
+  const stream = await navigator.mediaDevices.getUserMedia({
+    audio: false,
+    video: videoTrackConstraints,
+  });
+  test.trackForCleanup({
+    close() {
+      for (const track of stream.getTracks()) {
+        track.stop();
+      }
+    },
+  });
+  return stream;
+}
 
-  // Use MediaStreamTrackProcessor and ReadableStream to generate video frame directly.
-  if (typeof MediaStreamTrackProcessor !== 'undefined') {
-    const trackProcessor = new MediaStreamTrackProcessor({ track });
-    const reader = trackProcessor.readable.getReader();
-    const result = await reader.read();
-    if (result.done) {
-      test.skip('MediaStreamTrackProcessor: Cannot get valid frame from readable stream.');
+/**
+ * Chrome on macOS (at least) takes a while before it switches from blank frames
+ * to real frames. Wait up to 50 frames for something to show up on the camera.
+ */
+async function waitForNonBlankFrame({
+  getSource,
+  waitForNextFrame,
+}: {
+  getSource: () => HTMLVideoElement | VideoFrame;
+  waitForNextFrame: () => Promise<void>;
+}) {
+  const cvs = document.createElement('canvas');
+  [cvs.width, cvs.height] = [4, 4];
+  const ctx = cvs.getContext('2d', { willReadFrequently: true })!;
+  let foundNonBlankFrame = false;
+  for (let i = 0; i < 50; ++i) {
+    ctx.drawImage(getSource(), 0, 0, cvs.width, cvs.height);
+    const pixels = new Uint32Array(ctx.getImageData(0, 0, cvs.width, cvs.height).data.buffer);
+    // Look only at RGB, ignore alpha.
+    if (pixels.some(p => (p & 0x00ffffff) !== 0)) {
+      foundNonBlankFrame = true;
+      break;
     }
-
-    return result.value;
-  }
-
-  // Fallback to ImageCapture if MediaStreamTrackProcessor not supported. Using grabFrame() to
-  // generate imageBitmap and creating video frame from it.
-  if (typeof ImageCapture !== 'undefined') {
-    const imageCapture = new ImageCapture(track);
-    const imageBitmap = await imageCapture.grabFrame();
-    return new VideoFrame(imageBitmap);
+    await waitForNextFrame();
   }
+  assert(foundNonBlankFrame, 'Failed to get a non-blank video frame');
+}
 
-  // Fallback to using HTMLVideoElement to do capture.
+/**
+ * Uses MediaStreamTrackProcessor to capture a VideoFrame from the camera.
+ * Skips the test if not supported.
+ * @param videoTrackConstraints - MediaTrackConstraints (e.g. width/height) to pass to
+ *     `getUserMedia()`, or `true` if none.
+ */
+export async function getVideoFrameFromCamera(
+  test: Fixture,
+  videoTrackConstraints: MediaTrackConstraints | true
+): Promise<VideoFrame> {
   test.skipIf(
-    typeof HTMLVideoElement === 'undefined',
-    'Try to use HTMLVideoElement do capture but HTMLVideoElement not available.'
+    typeof MediaStreamTrackProcessor === 'undefined',
+    'MediaStreamTrackProcessor not supported'
   );
 
+  const stream = await getStreamFromCamera(test, videoTrackConstraints);
+  const tracks = stream.getVideoTracks();
+  assert(tracks.length > 0, 'no tracks found');
+  const track = tracks[0] as MediaStreamVideoTrack;
+
+  const trackProcessor = new MediaStreamTrackProcessor({ track });
+  const reader = trackProcessor.readable.getReader();
+
+  const waitForNextFrame = async () => {
+    const result = await reader.read();
+    assert(!result.done, "MediaStreamTrackProcessor: Couldn't get valid frame from stream.");
+    return result.value;
+  };
+  let frame: VideoFrame = await waitForNextFrame();
+  await waitForNonBlankFrame({
+    getSource: () => frame,
+    async waitForNextFrame() {
+      frame.close();
+      frame = await waitForNextFrame();
+    },
+  });
+
+  test.trackForCleanup(frame);
+  return frame;
+}
+
+/**
+ * Create an HTMLVideoElement from the camera stream. Skips the test if not supported.
+ * @param videoTrackConstraints - MediaTrackConstraints (e.g. width/height) to pass to
+ *     `getUserMedia()`, or `true` if none.
+ * @param paused - whether the video should be paused before returning.
+ */
+export async function getVideoElementFromCamera(
+  test: Fixture,
+  videoTrackConstraints: MediaTrackConstraints | true,
+  paused: boolean
+): Promise<HTMLVideoElement> {
+  const stream = await getStreamFromCamera(test, videoTrackConstraints);
+
+  // Main thread
   const video = document.createElement('video');
+  video.loop = false;
+  video.muted = true;
+  video.setAttribute('playsinline', '');
   video.srcObject = stream;
+  await new Promise(resolve => {
+    video.onloadedmetadata = resolve;
+  });
+  await startPlayingAndWaitForVideo(video, () => {});
 
-  const frame = await getVideoFrameFromVideoElement(test, video);
-  test.trackForCleanup(frame);
+  await waitForNonBlankFrame({
+    getSource: () => video,
+    waitForNextFrame: () =>
+      new Promise(resolve =>
+        video.requestVideoFrameCallback(() => {
+          resolve();
+        })
+      ),
+  });
 
-  return frame;
+  if (paused) {
+    // Pause the video so we get consistent readbacks.
+    await new Promise(resolve => {
+      video.onpause = resolve;
+      video.pause();
+    });
+  }
+
+  return video;
 }
 
 const kFourColorsInfo = {