feat: Add Frame.hasPixelBuffer and Frame.hasNativeBuffer (#3928)

* feat: Add `Frame.hasPixelBuffer` and `Frame.hasNativeBuffer`

* update docs

* chore: Allow skipping if unsupported

* more typed

Author: mrousavy

apps/simple-camera/__tests__/visioncamera.frame.harness.ts

@@ -76,7 +76,7 @@ describe('VisionCamera - Frame', () => {
     expect(framesReceived).toBeGreaterThanOrEqual(3)
   })
 
-  it('gets pixel buffers and releases native buffers from native frames', async () => {
+  it('reports and conditionally reads native frame buffers', async (context) => {
     const session = await VisionCamera.createCameraSession(false)
     const frameOutput = VisionCamera.createFrameOutput({
       targetResolution: CommonResolutions.HD_16_9,
@@ -95,59 +95,111 @@ describe('VisionCamera - Frame', () => {
       },
     ])
 
-    const receivedBuffers = deferred()
+    type NativeFrameBufferReport =
+      | { state: 'skip'; reason: string }
+      | { state: 'error'; errorMessage: string }
+      | { state: 'success' }
+
+    type NativeFrameBufferResult =
+      | { state: 'skip'; reason: string }
+      | { state: 'success'; frames: number }
+
+    const receivedBuffers = deferred<NativeFrameBufferResult>()
     let buffersReceived = 0
-    const report = (hasPixelBuffer: boolean, hasNativeBuffer: boolean) => {
-      if (!hasPixelBuffer) {
-        receivedBuffers.reject(new Error('Frame pixel buffer was empty.'))
-      } else if (!hasNativeBuffer) {
-        receivedBuffers.reject(new Error('Frame native buffer pointer was 0.'))
-      } else {
-        buffersReceived++
-        if (buffersReceived >= 3) {
-          receivedBuffers.resolve()
-        }
+    const report = (frameBufferReport: NativeFrameBufferReport) => {
+      switch (frameBufferReport.state) {
+        case 'skip':
+          receivedBuffers.resolve(frameBufferReport)
+          break
+        case 'error':
+          receivedBuffers.reject(new Error(frameBufferReport.errorMessage))
+          break
+        case 'success':
+          buffersReceived++
+          if (buffersReceived >= 3) {
+            receivedBuffers.resolve({
+              state: 'success',
+              frames: buffersReceived,
+            })
+          }
+          break
       }
     }
-    const reportError = (errorMessage: string) => {
-      receivedBuffers.reject(new Error(errorMessage))
-    }
     const errorSub = session.addOnErrorListener(receivedBuffers.reject)
 
     const runtime = workletsProvider.createRuntimeForThread(frameOutput.thread)
     runtime.setOnFrameCallback(frameOutput, (frame) => {
       'worklet'
       try {
-        const pixelBuffer = frame.getPixelBuffer()
-        const hasPixelBuffer = pixelBuffer.byteLength > 0
+        if (!frame.hasPixelBuffer) {
+          scheduleOnRN(report, {
+            state: 'skip',
+            reason:
+              'native frame buffers: device does not expose readable pixel buffers',
+          })
+          return
+        }
+
+        const pixelBufferBytes = frame.getPixelBuffer().byteLength
+        if (pixelBufferBytes <= 0) {
+          scheduleOnRN(report, {
+            state: 'error',
+            errorMessage: 'Frame pixel buffer was empty.',
+          })
+          return
+        }
+
+        if (!frame.hasNativeBuffer) {
+          scheduleOnRN(report, {
+            state: 'skip',
+            reason:
+              'native frame buffers: device does not expose native buffers',
+          })
+          return
+        }
+
         const nativeBuffer = frame.getNativeBuffer()
-        let hasNativeBuffer = false
         try {
-          hasNativeBuffer = nativeBuffer.pointer !== 0n
+          if (nativeBuffer.pointer === 0n) {
+            scheduleOnRN(report, {
+              state: 'error',
+              errorMessage: 'Frame native buffer pointer was 0.',
+            })
+            return
+          }
         } finally {
           nativeBuffer.release()
         }
-        scheduleOnRN(report, hasPixelBuffer, hasNativeBuffer)
+
+        scheduleOnRN(report, {
+          state: 'success',
+        })
       } catch (e) {
-        scheduleOnRN(reportError, String(e))
+        scheduleOnRN(report, {
+          state: 'error',
+          errorMessage: String(e),
+        })
       } finally {
         frame.dispose()
       }
     })
 
     await session.start()
     try {
-      await withTimeout(
+      const result = await withTimeout(
         receivedBuffers.promise,
         15_000,
-        'receive native frame pixel buffers',
+        'receive native frame buffer reports',
       )
+      if (result.state === 'skip') {
+        return context.skip(result.reason)
+      }
+      expect(result.frames).toBeGreaterThanOrEqual(3)
     } finally {
       runtime.setOnFrameCallback(frameOutput, undefined)
       errorSub.remove()
       await session.stop()
     }
-    expect(buffersReceived).toBeGreaterThanOrEqual(3)
   })
 
   it('keeps YUV plane buffers readable across repeated reads', async () => {
@@ -302,7 +354,7 @@ describe('VisionCamera - Frame', () => {
     expect(reportedPlanes).toBeGreaterThanOrEqual(1)
   })
 
-  it('delivers frames when streaming in rgb', async () => {
+  it('delivers readable pixel buffers when streaming in rgb', async () => {
     const session = await VisionCamera.createCameraSession(false)
     const frameOutput = VisionCamera.createFrameOutput({
       targetResolution: CommonResolutions.VGA_16_9,
@@ -322,16 +374,29 @@ describe('VisionCamera - Frame', () => {
     ])
 
     const receivedFrame = deferred()
-    const onFrame = () => {
-      receivedFrame.resolve()
+    const onFrame = (pixelBufferBytes: number) => {
+      if (pixelBufferBytes > 0) {
+        receivedFrame.resolve()
+      } else {
+        receivedFrame.reject(new Error('RGB frame pixel buffer was empty.'))
+      }
+    }
+    const onError = (errorMessage: string) => {
+      receivedFrame.reject(new Error(errorMessage))
     }
     const errorSub = session.addOnErrorListener(receivedFrame.reject)
 
     const runtime = workletsProvider.createRuntimeForThread(frameOutput.thread)
     runtime.setOnFrameCallback(frameOutput, (frame) => {
       'worklet'
-      scheduleOnRN(onFrame)
-      frame.dispose()
+      try {
+        const pixelBufferBytes = frame.getPixelBuffer().byteLength
+        scheduleOnRN(onFrame, pixelBufferBytes)
+      } catch (e) {
+        scheduleOnRN(onError, String(e))
+      } finally {
+        frame.dispose()
+      }
     })
 
     await session.start()

packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getNativeBuffer.kt

@@ -7,6 +7,18 @@ import androidx.camera.core.ImageProxy
 import com.margelo.nitro.camera.NativeBuffer
 import com.margelo.nitro.camera.utils.NativeBufferHelper
 
+val ImageProxy.hasNativeBuffer: Boolean
+  @OptIn(ExperimentalGetImage::class)
+  get() {
+    if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) {
+      return false
+    }
+    image?.hardwareBuffer?.use {
+      return true
+    }
+    return false
+  }
+
 @OptIn(ExperimentalGetImage::class)
 fun ImageProxy.getNativeBuffer(): NativeBuffer {
   if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) {

packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/extensions/ImageProxy+getPixelBuffer.kt

@@ -7,6 +7,7 @@ import androidx.camera.core.ExperimentalGetImage
 import androidx.camera.core.ImageProxy
 import com.margelo.nitro.camera.utils.DirectByteBufferPool
 import com.margelo.nitro.core.ArrayBuffer
+import java.nio.ByteBuffer
 
 val ImageProxy.hasPixelBuffer: Boolean
   @OptIn(ExperimentalGetImage::class)
@@ -30,6 +31,23 @@ data class DisposableArrayBuffer(
   val dispose: () -> Unit,
 )
 
+private fun ByteBuffer.wrapOrCopyIntoArrayBuffer(): DisposableArrayBuffer {
+  val buffer = readableBytes()
+  if (buffer.isDirect) {
+    val arrayBuffer = ArrayBuffer.wrap(buffer)
+    return DisposableArrayBuffer(arrayBuffer) {
+      // no release
+    }
+  }
+
+  val directBuffer = DirectByteBufferPool.Shared.acquire(buffer.remaining())
+  directBuffer.put(buffer)
+  val arrayBuffer = ArrayBuffer.wrap(directBuffer)
+  return DisposableArrayBuffer(arrayBuffer) {
+    DirectByteBufferPool.Shared.release(directBuffer)
+  }
+}
+
 @OptIn(ExperimentalGetImage::class)
 fun ImageProxy.getPixelBuffer(): DisposableArrayBuffer {
   if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
@@ -47,12 +65,8 @@ fun ImageProxy.getPixelBuffer(): DisposableArrayBuffer {
   }
   when {
     planes.size == 1 -> {
-      // Medium Path: We can wrap a single plane as a ByteBuffer
-      val byteBuffer = planes.single().buffer.readableBytes()
-      val arrayBuffer = ArrayBuffer.wrap(byteBuffer)
-      return DisposableArrayBuffer(arrayBuffer) {
-        // no release
-      }
+      // Medium Path: We can wrap a single direct plane as a ByteBuffer, or copy it into one if needed.
+      return planes.single().buffer.wrapOrCopyIntoArrayBuffer()
     }
     planes.size > 1 -> {
       // Slow Path: We have to copy all planes into a new ByteBuffer.

packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/instances/HybridFrame.kt

@@ -12,6 +12,8 @@ import com.margelo.nitro.camera.extensions.DisposableArrayBuffer
 import com.margelo.nitro.camera.extensions.convertPoint
 import com.margelo.nitro.camera.extensions.getNativeBuffer
 import com.margelo.nitro.camera.extensions.getPixelBuffer
+import com.margelo.nitro.camera.extensions.hasNativeBuffer
+import com.margelo.nitro.camera.extensions.hasPixelBuffer
 import com.margelo.nitro.camera.extensions.mapToArray
 import com.margelo.nitro.camera.extensions.pixelFormat
 import com.margelo.nitro.camera.public.NativeFrame
@@ -50,6 +52,10 @@ class HybridFrame(
     get() = image.pixelFormat
   override val isPlanar: Boolean
     get() = image.planes.size > 1
+  override val hasPixelBuffer: Boolean
+    get() = image.hasPixelBuffer
+  override val hasNativeBuffer: Boolean
+    get() = image.hasNativeBuffer
 
   // TODO: Implement `cameraIntrinsicMatrix`
   override val cameraIntrinsicMatrix: DoubleArray?

packages/react-native-vision-camera/ios/Hybrid Objects/Image Types/HybridFrame.swift

@@ -74,6 +74,14 @@ final class HybridFrame: HybridFrameSpec, NativeFrame, LazyLockableBuffer {
     return CVPixelBufferIsPlanar(pixelBuffer)
   }
 
+  var hasPixelBuffer: Bool {
+    return pixelBuffer != nil
+  }
+
+  var hasNativeBuffer: Bool {
+    return pixelBuffer != nil
+  }
+
   var pixelFormat: PixelFormat {
     guard let pixelBuffer else {
       return .unknown