🔧

Author: wcandillon

apps/example/src/SharedTextureMemory/SharedTextureMemory.tsx

@@ -4,7 +4,9 @@ import {
   Canvas,
   useCanvasRef,
   useDevice,
+  type GPUSharedTextureMemory,
   type NativeCanvas,
+  type VideoFrame,
 } from "react-native-wgpu";
 
 const SHADER = /* wgsl */ `
@@ -41,37 +43,44 @@ fn fs_main(in: VsOut) -> @location(0) vec4f {
 }
 `;
 
-const REQUIRED_FEATURE =
+// On Metal, EndAccess on an IOSurface-backed SharedTextureMemory always
+// produces an MTLSharedEvent fence (so the producer can wait on the GPU). Even
+// though we don't currently expose the fence to JS, Dawn validates that the
+// fence feature is enabled before letting EndAccess succeed. Android has the
+// equivalent pairing with sync fds.
+const REQUIRED_FEATURES =
   Platform.OS === "ios"
-    ? "shared-texture-memory-iosurface"
-    : "shared-texture-memory-ahardware-buffer";
+    ? ["shared-texture-memory-iosurface", "shared-fence-mtl-shared-event"]
+    : [
+        "shared-texture-memory-ahardware-buffer",
+        "shared-fence-vk-semaphore-sync-fd",
+      ];
 
 export const SharedTextureMemory = () => {
   const ref = useCanvasRef();
   const [error, setError] = useState<string | null>(null);
   const rafRef = useRef<number | null>(null);
 
-  // Request the shared-memory feature when constructing the device so the
-  // shared-texture-memory* extension is enabled.
   const { device, adapter } = useDevice(undefined, {
     // Cast: GPUFeatureName in @webgpu/types doesn't include the Dawn-specific
-    // extension name yet, but Dawn accepts it.
-    requiredFeatures: [REQUIRED_FEATURE as GPUFeatureName],
+    // extension names yet, but Dawn accepts them.
+    requiredFeatures: REQUIRED_FEATURES as unknown as GPUFeatureName[],
   });
 
   useEffect(() => {
     if (!device) {
       return;
     }
-    if (!device.features.has(REQUIRED_FEATURE)) {
+    const missing = REQUIRED_FEATURES.filter((f) => !device.features.has(f));
+    if (missing.length > 0) {
       setError(
-        `Device is missing the '${REQUIRED_FEATURE}' feature (adapter supports: ${
+        `Device is missing required features [${missing.join(", ")}]. Adapter supports: ${
           adapter
             ? [...adapter.features]
                 .filter((f) => f.toString().startsWith("shared-"))
                 .join(", ") || "none"
             : "n/a"
-        })`,
+        }`,
       );
       return;
     }
@@ -90,38 +99,14 @@ export const SharedTextureMemory = () => {
       alphaMode: "premultiplied",
     });
 
-    // 1. Acquire a native, GPU-shareable surface. In production this would
-    //    come from a camera frame processor or video decoder. The test helper
-    //    synthesizes a 256x256 RGB-gradient pattern in an IOSurface.
-    const frame = RNWebGPU.createTestVideoFrame(256, 256);
-
-    // 2. Import the raw native handle into a SharedTextureMemory.
-    const sharedMemory = device.importSharedTextureMemory({
-      handle: frame.handle,
-      label: "video-frame-shared-memory",
-    });
-
-    // 3. Create a regular GPUTexture that aliases the surface's pixels.
-    //    No descriptor needed: the format/size are inferred from the surface.
-    const texture = sharedMemory.createTexture();
-
-    // 4. beginAccess declares that we're about to read or write the texture on
-    //    the GPU timeline. `initialized: true` means "the surface already has
-    //    meaningful pixels", which is correct for an incoming video frame.
-    //
-    //    Because this example owns a *static* IOSurface (no external producer
-    //    is writing new pixels between frames), we keep one access window open
-    //    for the lifetime of the texture and call endAccess only on unmount.
-    //
-    //    For a live camera or video feed, you'd instead wrap each frame:
-    //      beginAccess(tex, true) -> submit -> endAccess(tex)
-    //    around every render to hand ownership back to the producer. That's
-    //    also where fence support (not yet wired through this binding) becomes
-    //    important to avoid races with the producer.
-    if (!sharedMemory.beginAccess(texture, true)) {
-      setError("beginAccess() failed");
-      return;
-    }
+    // 1. Open the video and start playback. AVPlayer accepts local file paths
+    //    as well as http(s):// URLs and keeps the IOSurface pool up to date
+    //    in the background. For a fully offline demo, swap this URL for
+    //    RNWebGPU.writeTestVideoFile() which generates a tiny mp4 on disk.
+    const VIDEO_URL =
+      "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/1080/Big_Buck_Bunny_1080_10s_5MB.mp4";
+    const player = RNWebGPU.createVideoPlayer(VIDEO_URL);
+    player.play();
 
     const module = device.createShaderModule({ code: SHADER });
     const pipeline = device.createRenderPipeline({
@@ -138,15 +123,77 @@ export const SharedTextureMemory = () => {
       magFilter: "linear",
       minFilter: "linear",
     });
-    const bindGroup = device.createBindGroup({
-      layout: pipeline.getBindGroupLayout(0),
-      entries: [
-        { binding: 0, resource: texture.createView() },
-        { binding: 1, resource: sampler },
-      ],
-    });
+
+    // We hold the *current* frame across rAF ticks so that when the video
+    // hasn't produced a new frame yet (between decoder timestamps), we keep
+    // rendering the last one rather than dropping to a black screen.
+    //
+    // For each new IOSurface we:
+    //   - create a SharedTextureMemory + texture + bindGroup
+    //   - beginAccess(initialized: true) to declare "the producer has written
+    //     these pixels and we're now sampling them"
+    //   - sample in the shader
+    //   - endAccess to hand ownership back to the producer
+    //
+    // We close out the previous frame's access window first. In a fence-aware
+    // build we'd plumb an AVPlayer fence through beginAccess/endAccess; for the
+    // demo we rely on AVPlayer recycling its IOSurface pool, which is safe as
+    // long as we end-access before letting the player reclaim the buffer.
+    type Bound = {
+      frame: VideoFrame;
+      memory: GPUSharedTextureMemory;
+      texture: GPUTexture;
+      bindGroup: GPUBindGroup;
+    };
+    let current: Bound | null = null;
+
+    const bindFrame = (frame: VideoFrame): Bound | null => {
+      try {
+        const memory = device.importSharedTextureMemory({
+          handle: frame.handle,
+          label: "video-frame",
+        });
+        const texture = memory.createTexture();
+        if (!memory.beginAccess(texture, true)) {
+          texture.destroy();
+          frame.release();
+          return null;
+        }
+        const bindGroup = device.createBindGroup({
+          layout: pipeline.getBindGroupLayout(0),
+          entries: [
+            { binding: 0, resource: texture.createView() },
+            { binding: 1, resource: sampler },
+          ],
+        });
+        return { frame, memory, texture, bindGroup };
+      } catch (e) {
+        console.warn("[SharedTextureMemory] bindFrame failed:", e);
+        frame.release();
+        return null;
+      }
+    };
+
+    const releaseBound = (b: Bound) => {
+      b.memory.endAccess(b.texture);
+      b.texture.destroy();
+      b.frame.release();
+    };
 
     const render = () => {
+      // Pull the latest frame from the player. Null means "no new frame since
+      // we last asked", in which case we keep using the existing one.
+      const newFrame = player.copyLatestFrame();
+      if (newFrame) {
+        const next = bindFrame(newFrame);
+        if (next) {
+          if (current) {
+            releaseBound(current);
+          }
+          current = next;
+        }
+      }
+
       const encoder = device.createCommandEncoder();
       const pass = encoder.beginRenderPass({
         colorAttachments: [
@@ -158,9 +205,11 @@ export const SharedTextureMemory = () => {
           },
         ],
       });
-      pass.setPipeline(pipeline);
-      pass.setBindGroup(0, bindGroup);
-      pass.draw(3);
+      if (current) {
+        pass.setPipeline(pipeline);
+        pass.setBindGroup(0, current.bindGroup);
+        pass.draw(3);
+      }
       pass.end();
       device.queue.submit([encoder.finish()]);
       context.present();
@@ -172,9 +221,11 @@ export const SharedTextureMemory = () => {
       if (rafRef.current !== null) {
         cancelAnimationFrame(rafRef.current);
       }
-      sharedMemory.endAccess(texture);
-      texture.destroy();
-      frame.release();
+      if (current) {
+        releaseBound(current);
+        current = null;
+      }
+      player.release();
     };
   }, [device, adapter, ref]);
 

packages/webgpu/android/cpp/AndroidPlatformContext.h

@@ -219,6 +219,19 @@ class AndroidPlatformContext : public PlatformContext {
     throw std::runtime_error(
         "createTestVideoFrame is not yet implemented on Android.");
   }
+
+  std::unique_ptr<IVideoPlayer>
+  createVideoPlayer(const std::string & /*path*/) override {
+    // TODO: implement using MediaCodec -> ImageReader (AHardwareBuffer mode).
+    throw std::runtime_error(
+        "createVideoPlayer is not yet implemented on Android.");
+  }
+
+  std::string writeTestVideoFile() override {
+    // TODO: implement using MediaCodec (H.264 encoder) or MediaMuxer.
+    throw std::runtime_error(
+        "writeTestVideoFile is not yet implemented on Android.");
+  }
 };
 
 } // namespace rnwgpu

packages/webgpu/apple/ApplePlatformContext.h

@@ -31,6 +31,11 @@ class ApplePlatformContext : public PlatformContext {
 
   VideoFrameHandle createTestVideoFrame(uint32_t width,
                                         uint32_t height) override;
+
+  std::unique_ptr<IVideoPlayer>
+  createVideoPlayer(const std::string &path) override;
+
+  std::string writeTestVideoFile() override;
 };
 
 } // namespace rnwgpu

packages/webgpu/apple/ApplePlatformContext.mm

@@ -8,6 +8,8 @@
 #import <React/RCTBridge+Private.h>
 #import <ReactCommon/RCTTurboModule.h>
 
+#include "AppleVideoPlayer.h"
+
 #include "RNWebGPUManager.h"
 #include "WebGPUModule.h"
 
@@ -234,6 +236,15 @@ void checkIfUsingSimulatorWithAPIValidation() {
   return handle;
 }
 
+std::unique_ptr<IVideoPlayer>
+ApplePlatformContext::createVideoPlayer(const std::string &path) {
+  return createAppleVideoPlayer(path);
+}
+
+std::string ApplePlatformContext::writeTestVideoFile() {
+  return writeAppleTestVideoFile();
+}
+
 VideoFrameHandle
 ApplePlatformContext::createTestVideoFrame(uint32_t width, uint32_t height) {
   NSDictionary *attrs = @{

packages/webgpu/apple/AppleVideoPlayer.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "PlatformContext.h"
+
+#include <memory>
+#include <string>
+
+namespace rnwgpu {
+
+// Factory: creates a new IVideoPlayer backed by AVPlayer +
+// AVPlayerItemVideoOutput.
+std::unique_ptr<IVideoPlayer>
+createAppleVideoPlayer(const std::string &path);
+
+// Generate a small procedurally-animated test video and write it to a
+// temporary file. Returns the absolute path. Used by the SharedTextureMemory
+// example so it doesn't need a bundled .mp4.
+std::string writeAppleTestVideoFile();
+
+} // namespace rnwgpu