feat(🤖): correct per-buffer YUV conversion for Android camera frames (#387)

Three improvements to the Android camera-frame path, found while reviewing
the VisionCamera integration:

1. Expose GPUExternalTexture.yuvToRgbMatrix (non-spec extension).
   Dawn samples Android external-format (opaque YCbCr) buffers through a
   Vulkan conversion hard-coded to RGB_IDENTITY (SamplerVk.cpp, see
   crbug.com/497675620), so shaders receive raw [Y, Cb, Cr] and previously
   had to guess the conversion. The example hard-coded BT.709 narrow-range,
   but Android camera streams are usually BT.601, and range varies by
   device, which skews colors. The driver reports the correct model and
   range per buffer (suggestedYcbcrModel/Range); Dawn captures them at
   import time and we now read them back via the shared memory's
   AHardwareBuffer properties and derive the exact 3x4 conversion matrix.
   On iOS and for RGBA surfaces the matrix is the identity passthrough, so
   shaders can apply it unconditionally. The VisionCamera example now
   uploads the matrix as uniforms instead of hard-coding coefficients.

2. Fail fast on CPU-only AHardwareBuffers.
   Dawn only grants TextureBinding when the AHB was allocated with
   AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE; without it, import fails deep
   inside Dawn with an opaque validation error. wrapNativeBuffer now checks
   the usage bits and throws an actionable error (pointing at
   vision-camera's pixelFormat: 'native').

3. Remove the unreachable "defined biplanar format" branch on Android.
   Dawn maps every YUV AHB format (Y8Cb8Cr8_420, P010, vendor formats) to
   OpaqueYCbCrAndroid (AHBFunctions.cpp), so the R8BG8Biplanar420Unorm
   plane-splitting path and its BT.709 matrix could never run.

Also folds the Android both-axes flip in the example into the rotation
passed to importExternalTexture (a double mirror is a 180° rotation),
removing the per-fragment flip branches from the WGSL prelude.

Author: wcandillon

apps/example/src/VisionCamera/VisionCamera.tsx

@@ -167,6 +167,8 @@ const CameraView = () => {
   // This is what lets us call the interop factory off RNWebGPU, where the native
   // platform context already lives, instead of off `device`.
   const rnwgpu = RNWebGPU;
+  // Captured as a plain bool so the worklet doesn't reach for Platform.
+  const isAndroid = Platform.OS === "android";
   const devices = useCameraDevices();
   // Pick back camera if available, otherwise front, otherwise anything. The
   // iOS simulator returns an empty list since there are no cameras, in which
@@ -254,7 +256,8 @@ const CameraView = () => {
       minFilter: "linear",
     });
     const uniformBuffer = device.createBuffer({
-      size: 32,
+      // vec4f params + vec4u modes + 3x vec4f yuvToRgbMatrix rows.
+      size: 80,
       usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
     });
 
@@ -282,7 +285,8 @@ const CameraView = () => {
       primitive: { topology: "triangle-list" },
     });
     const prepassUniformBuffer = device.createBuffer({
-      size: 16,
+      // 2x vec2f sizes + 3x vec4f yuvToRgbMatrix rows.
+      size: 64,
       usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
     });
 
@@ -469,8 +473,11 @@ const CameraView = () => {
           // Orientation. The sensor buffer is landscape; frame.orientation is
           // the rotation needed to bring it upright. We hand that to Dawn via
           // importExternalTexture's `rotation`, which de-rotates the frame into
-          // the portrait canvas. The residual vertical flip (Android buffer
-          // Y-origin) and YUV->RGB are corrected in the shader (CAMERA_PRELUDE).
+          // the portrait canvas. On Android the buffer additionally arrives
+          // mirrored on both axes relative to the canvas (buffer Y-origin),
+          // which is the same as an extra 180° rotation, folded in below.
+          // YUV->RGB is applied in the shader via the
+          // GPUExternalTexture.yuvToRgbMatrix uniform (see CAMERA_PRELUDE).
           let rotationDeg: 0 | 90 | 180 | 270 = 0;
           if (frame.orientation === "right") {
             rotationDeg = 90;
@@ -479,6 +486,9 @@ const CameraView = () => {
           } else if (frame.orientation === "left") {
             rotationDeg = 270;
           }
+          if (isAndroid) {
+            rotationDeg = ((rotationDeg + 180) % 360) as 0 | 90 | 180 | 270;
+          }
           // A 90/270 rotation swaps the displayed width & height, so cover-fit
           // uses the post-rotation dimensions.
           const rotated = rotationDeg === 90 || rotationDeg === 270;
@@ -496,21 +506,6 @@ const CameraView = () => {
           } else {
             sy = frameAR / canvasAR;
           }
-          // 32-byte uniform: vec4f params + vec4u modes. Built on a single
-          // ArrayBuffer so the f32/u32 halves go up in one writeBuffer call.
-          const uniformData = new ArrayBuffer(32);
-          const uniformF32 = new Float32Array(uniformData);
-          const uniformU32 = new Uint32Array(uniformData);
-          uniformF32[0] = sx;
-          uniformF32[1] = sy;
-          uniformF32[2] = ABERRATION_STRENGTHS[modes.aberration] ?? 0;
-          uniformF32[3] = PIXELATE_BLOCKS[modes.pixelate] ?? 0;
-          uniformU32[4] = modes.effect;
-          uniformU32[5] = modes.tint;
-          uniformU32[6] = modes.vignette;
-          uniformU32[7] = blurMode;
-          device.queue.writeBuffer(uniformBuffer, 0, uniformData);
-
           let externalTex;
           try {
             externalTex = device.importExternalTexture({
@@ -525,6 +520,29 @@ const CameraView = () => {
             );
             throw e;
           }
+          // Per-buffer YUV->RGB conversion for the shader: the driver-derived
+          // matrix on the Android opaque-YCbCr path, an identity passthrough
+          // on iOS / RGBA surfaces (see CAMERA_PRELUDE).
+          const yuvMatrix = externalTex.yuvToRgbMatrix;
+
+          // 80-byte uniform: vec4f params + vec4u modes + the three vec4f
+          // yuvToRgbMatrix rows. Built on a single ArrayBuffer so the f32/u32
+          // halves go up in one writeBuffer call.
+          const uniformData = new ArrayBuffer(80);
+          const uniformF32 = new Float32Array(uniformData);
+          const uniformU32 = new Uint32Array(uniformData);
+          uniformF32[0] = sx;
+          uniformF32[1] = sy;
+          uniformF32[2] = ABERRATION_STRENGTHS[modes.aberration] ?? 0;
+          uniformF32[3] = PIXELATE_BLOCKS[modes.pixelate] ?? 0;
+          uniformU32[4] = modes.effect;
+          uniformU32[5] = modes.tint;
+          uniformU32[6] = modes.vignette;
+          uniformU32[7] = blurMode;
+          for (let i = 0; i < 12; i++) {
+            uniformF32[8 + i] = yuvMatrix[i] ?? 0;
+          }
+          device.queue.writeBuffer(uniformBuffer, 0, uniformData);
           const bindGroup = device.createBindGroup({
             layout: pipeline.getBindGroupLayout(0),
             entries: [
@@ -542,7 +560,13 @@ const CameraView = () => {
             device.queue.writeBuffer(
               prepassUniformBuffer,
               0,
-              new Float32Array([dispW, dispH, canvasWidth, canvasHeight]),
+              new Float32Array([
+                dispW,
+                dispH,
+                canvasWidth,
+                canvasHeight,
+                ...yuvMatrix,
+              ]),
             );
             const prepassBindGroup = device.createBindGroup({
               layout: prepassPipeline.getBindGroupLayout(0),

apps/example/src/VisionCamera/blurShaders.ts

@@ -21,6 +21,10 @@ export const PREPASS_SHADER = /* wgsl */ `
 struct PrepassUniforms {
   texSize: vec2f,
   canvasSize: vec2f,
+  // The three rows of GPUExternalTexture.yuvToRgbMatrix (see CAMERA_PRELUDE).
+  yuv0: vec4f,
+  yuv1: vec4f,
+  yuv2: vec4f,
 };
 
 @group(0) @binding(0) var srcTex: texture_external;
@@ -63,14 +67,14 @@ fn fs_main(in: VsOut) -> @location(0) vec4f {
     scale = vec2f(1.0, texAR / canvasAR);
   }
   let uv = vec2f(0.5) + (in.uv - vec2f(0.5)) * scale;
-  // cameraCoord (vertical flip) + cameraDecode (YUV->RGB) come from
-  // CAMERA_PRELUDE, prepended when this module is compiled. They are no-ops on
-  // iOS and handle the Android opaque-YUV case.
+  // cameraDecode (from CAMERA_PRELUDE, prepended when this module is
+  // compiled) applies GPUExternalTexture.yuvToRgbMatrix: the real YUV->RGB
+  // conversion on the Android opaque-YUV path, an identity passthrough on iOS.
   let c = cameraDecode(textureSampleBaseClampToEdge(
     srcTex,
     srcSampler,
-    cameraCoord(clamp(uv, vec2f(0.0), vec2f(1.0))),
-  ));
+    clamp(uv, vec2f(0.0), vec2f(1.0)),
+  ), u.yuv0, u.yuv1, u.yuv2);
   return vec4f(c.rgb, 1.0);
 }
 `;

apps/example/src/VisionCamera/shaders.ts

@@ -1,10 +1,8 @@
-import { Platform } from "react-native";
-
 // Main camera-effects shader: samples the imported external texture (camera
 // frame) with cover-fit + optional chromatic aberration / pixelate, optionally
 // mixes in the pre-blurred backdrop, then applies effect / tint / vignette.
-// `cameraCoord` / `cameraDecode` come from CAMERA_PRELUDE, which is prepended
-// at shader-module creation time.
+// `cameraDecode` comes from CAMERA_PRELUDE, which is prepended at
+// shader-module creation time.
 export const SHADER = /* wgsl */ `
 struct VsOut {
   @builtin(position) position: vec4f,
@@ -22,6 +20,10 @@ struct Uniforms {
   // w: blurMode (0 off, 1 strong - blurred everywhere (prepass bakes
   //              cover-fit), 2 overlay - blurred backdrop + sharp card)
   modes: vec4u,
+  // The three rows of GPUExternalTexture.yuvToRgbMatrix (see CAMERA_PRELUDE).
+  yuv0: vec4f,
+  yuv1: vec4f,
+  yuv2: vec4f,
 };
 
 @group(0) @binding(0) var srcTex: texture_external;
@@ -56,7 +58,8 @@ fn snap(uv: vec2f, block: f32) -> vec2f {
 
 fn sampleExternal(uv: vec2f, block: f32) -> vec4f {
   return cameraDecode(
-    textureSampleBaseClampToEdge(srcTex, srcSampler, cameraCoord(snap(uv, block))),
+    textureSampleBaseClampToEdge(srcTex, srcSampler, snap(uv, block)),
+    u.yuv0, u.yuv1, u.yuv2,
   );
 }
 
@@ -177,43 +180,21 @@ fn fs_main(in: VsOut) -> @location(0) vec4f {
 // YCbCr conversion that is hard-coded to RGB_IDENTITY (Dawn's
 // SamplerVk.cpp::GetYCbCrForTextureView; see crbug.com/497675620), so the
 // external sample comes back as raw [Y, Cb, Cr] on *every* device — this is by
-// design, not a driver quirk — and we do the BT.709 YUV->RGB ourselves below.
-// (Dawn's own SharedTextureMemoryOpaqueYCbCrAndroidForExternalTexture
-// .NoopSampleY8Cb8Cr8AHB test asserts the same raw passthrough.) The frame also
-// comes out mirrored on both axes relative to the canvas (Android buffer
-// origin), so we flip X and Y. iOS goes through the native two-plane path,
-// which already converts and orients, so this correction is Android-only. The
-// prelude is prepended to every shader module that samples the camera (main
-// pass + blur prepass).
+// design, not a driver quirk. The correct conversion depends on the buffer:
+// react-native-webgpu derives it from the driver's suggested YCbCr model and
+// range (BT.601/709/2020, full/narrow) and exposes it as
+// GPUExternalTexture.yuvToRgbMatrix; the worklet uploads its three rows as
+// vec4f uniforms and cameraDecode applies them. On iOS (native two-plane path,
+// already converted by Dawn) and for RGBA surfaces the matrix is the identity
+// passthrough, so the decode is safe to apply unconditionally. The prelude is
+// prepended to every shader module that samples the camera (main pass + blur
+// prepass).
 export const CAMERA_PRELUDE = /* wgsl */ `
-const CAMERA_IS_YUV: bool = ${Platform.OS === "android"};
-const CAMERA_FLIP_X: bool = ${Platform.OS === "android"};
-const CAMERA_FLIP_Y: bool = ${Platform.OS === "android"};
-
-fn cameraCoord(uv: vec2f) -> vec2f {
-  var c = uv;
-  if (CAMERA_FLIP_X) {
-    c.x = 1.0 - c.x;
-  }
-  if (CAMERA_FLIP_Y) {
-    c.y = 1.0 - c.y;
-  }
-  return c;
-}
-
-// BT.709 limited-range YUV -> RGB. On the Android opaque path the sampled
-// channels are always raw [Y, Cb, Cr] (Dawn forces an RGB_IDENTITY Vulkan
-// conversion); a no-op passthrough on every other platform.
-fn cameraDecode(c: vec4f) -> vec4f {
-  if (!CAMERA_IS_YUV) {
-    return c;
-  }
-  let y = c.r - 0.0627451;
-  let cb = c.g - 0.5;
-  let cr = c.b - 0.5;
-  let r = 1.164384 * y + 1.792741 * cr;
-  let g = 1.164384 * y - 0.213249 * cb - 0.532909 * cr;
-  let b = 1.164384 * y + 2.112402 * cb;
-  return vec4f(clamp(vec3f(r, g, b), vec3f(0.0), vec3f(1.0)), 1.0);
+// Apply a 3x4 row-major [c.r, c.g, c.b, 1] -> R'G'B' matrix
+// (GPUExternalTexture.yuvToRgbMatrix, see above).
+fn cameraDecode(c: vec4f, m0: vec4f, m1: vec4f, m2: vec4f) -> vec4f {
+  let v = vec4f(c.rgb, 1.0);
+  let rgb = vec3f(dot(m0, v), dot(m1, v), dot(m2, v));
+  return vec4f(clamp(rgb, vec3f(0.0), vec3f(1.0)), 1.0);
 }
 `;

packages/webgpu/android/cpp/AndroidPlatformContext.h

@@ -241,8 +241,8 @@ class AndroidPlatformContext : public PlatformContext {
     const uint32_t stridePixels = actualDesc.stride;
 
     void *vaddr = nullptr;
-    int rc = AHardwareBuffer_lock(buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY,
-                                  -1, nullptr, &vaddr);
+    int rc = AHardwareBuffer_lock(
+        buffer, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY, -1, nullptr, &vaddr);
     if (rc != 0 || !vaddr) {
       AHardwareBuffer_release(buffer);
       throw std::runtime_error(
@@ -302,6 +302,21 @@ class AndroidPlatformContext : public PlatformContext {
     AHardwareBuffer_Desc desc = {};
     AHardwareBuffer_describe(buffer, &desc);
 
+    // Dawn derives the importable WebGPU usages from the AHB's usage bits;
+    // without GPU_SAMPLED_IMAGE the imported texture never gets
+    // TextureBinding and import fails deep inside Dawn with an opaque
+    // validation error. Surface the real cause here instead. (CameraX's
+    // default ImageReaders allocate CPU-only buffers; with
+    // react-native-vision-camera, use pixelFormat: 'native' which allocates
+    // GPU-sampleable buffers.)
+    if ((desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) == 0) {
+      throw std::runtime_error(
+          "wrapNativeBuffer: this AHardwareBuffer was allocated without "
+          "AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, so the GPU cannot sample "
+          "it. Camera/CPU pipelines must allocate frames with GPU usage (for "
+          "react-native-vision-camera, use pixelFormat: 'native').");
+    }
+
     AHardwareBuffer_acquire(buffer);
 
     VideoFrameHandle handle;
@@ -312,13 +327,13 @@ class AndroidPlatformContext : public PlatformContext {
     // Dawn's OpaqueYCbCrAndroidForExternalTexture path. Single-plane RGBA AHBs
     // take the plain BGRA8 path (sampled as a regular 2D texture).
     switch (desc.format) {
-      case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420:
-      case AHARDWAREBUFFER_FORMAT_YCbCr_P010:
-        handle.pixelFormat = VideoPixelFormat::NV12;
-        break;
-      default:
-        handle.pixelFormat = VideoPixelFormat::BGRA8;
-        break;
+    case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420:
+    case AHARDWAREBUFFER_FORMAT_YCbCr_P010:
+      handle.pixelFormat = VideoPixelFormat::NV12;
+      break;
+    default:
+      handle.pixelFormat = VideoPixelFormat::BGRA8;
+      break;
     }
     handle.deleter = [buffer]() { AHardwareBuffer_release(buffer); };
     return handle;