Skip to content

Commit bb74056

Browse files
wcandillonwcandillon
committed
🔧
1 parent 6f9b11a commit bb74056

6 files changed

Lines changed: 116 additions & 20 deletions

File tree

‎apps/example/src/VisionCamera/VisionCamera.tsx‎

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ import React, { useEffect } from "react";
22
import {
33
Linking,
44
PixelRatio,
5-
Platform,
65
StyleSheet,
76
Text,
87
TouchableOpacity,
@@ -80,18 +79,10 @@ fn fs_main(in: VsOut) -> @location(0) vec4f {
8079
}
8180
`;
8281

83-
const REQUIRED_FEATURES =
84-
Platform.OS === "ios"
85-
? [
86-
"shared-texture-memory-iosurface",
87-
"shared-fence-mtl-shared-event",
88-
"dawn-multi-planar-formats",
89-
]
90-
: [
91-
"shared-texture-memory-ahardware-buffer",
92-
"shared-fence-vk-semaphore-sync-fd",
93-
"dawn-multi-planar-formats",
94-
];
82+
const REQUIRED_FEATURES: GPUFeatureName[] = [
83+
"rnwebgpu/shared-texture-memory" as GPUFeatureName,
84+
"dawn-multi-planar-formats" as GPUFeatureName,
85+
];
9586

9687
const ABERRATION_STRENGTH = 0.006;
9788

@@ -124,7 +115,7 @@ export const VisionCamera = () => {
124115
const CameraView = () => {
125116
const ref = useCanvasRef();
126117
const { device, adapter } = useDevice(undefined, {
127-
requiredFeatures: REQUIRED_FEATURES as unknown as GPUFeatureName[],
118+
requiredFeatures: REQUIRED_FEATURES,
128119
});
129120
const devices = useCameraDevices();
130121
// Pick back camera if available, otherwise front, otherwise anything. The

‎packages/webgpu/android/cpp/AndroidPlatformContext.h‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -293,11 +293,35 @@ class AndroidPlatformContext : public PlatformContext {
293293
"writeTestVideoFile is not yet implemented on Android.");
294294
}
295295

296-
VideoFrameHandle wrapNativeBuffer(void * /*pointer*/) override {
297-
// TODO: AHardwareBuffer_acquire + extract dimensions, format, color
298-
// metadata.
299-
throw std::runtime_error(
300-
"wrapNativeBuffer is not yet implemented on Android.");
296+
VideoFrameHandle wrapNativeBuffer(void *pointer) override {
297+
if (!pointer) {
298+
throw std::runtime_error("wrapNativeBuffer: pointer is null");
299+
}
300+
auto *buffer = static_cast<AHardwareBuffer *>(pointer);
301+
302+
AHardwareBuffer_Desc desc = {};
303+
AHardwareBuffer_describe(buffer, &desc);
304+
305+
AHardwareBuffer_acquire(buffer);
306+
307+
VideoFrameHandle handle;
308+
handle.handle = static_cast<void *>(buffer);
309+
handle.width = desc.width;
310+
handle.height = desc.height;
311+
// YUV / opaque formats route through Vulkan's SamplerYcbcrConversion via
312+
// Dawn's OpaqueYCbCrAndroidForExternalTexture path. Single-plane RGBA AHBs
313+
// take the plain BGRA8 path (sampled as a regular 2D texture).
314+
switch (desc.format) {
315+
case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420:
316+
case AHARDWAREBUFFER_FORMAT_YCbCr_P010:
317+
handle.pixelFormat = VideoPixelFormat::NV12;
318+
break;
319+
default:
320+
handle.pixelFormat = VideoPixelFormat::BGRA8;
321+
break;
322+
}
323+
handle.deleter = [buffer]() { AHardwareBuffer_release(buffer); };
324+
return handle;
301325
}
302326
};
303327

‎packages/webgpu/cpp/rnwgpu/api/GPUDevice.cpp‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,74 @@ std::shared_ptr<GPUExternalTexture> GPUDevice::importExternalTexture(
363363
"null");
364364
}
365365

366+
return std::make_shared<GPUExternalTexture>(
367+
std::move(external), std::move(memory), std::move(texture),
368+
std::move(descriptor->source), std::move(label));
369+
#elif defined(__ANDROID__)
370+
// 1. Import the AHardwareBuffer as SharedTextureMemory. For YUV AHBs this
371+
// yields a Dawn texture in the implementation-defined OpaqueYCbCrAndroid
372+
// format; for RGBA AHBs, a regular single-plane texture.
373+
wgpu::SharedTextureMemoryDescriptor memDesc{};
374+
std::string label = descriptor->label.value_or("external-texture");
375+
if (!label.empty()) {
376+
memDesc.label = wgpu::StringView(label.c_str(), label.size());
377+
}
378+
wgpu::SharedTextureMemoryAHardwareBufferDescriptor platformDesc{};
379+
platformDesc.handle = frame.handle;
380+
memDesc.nextInChain = &platformDesc;
381+
auto memory = _instance.ImportSharedTextureMemory(&memDesc);
382+
if (memory == nullptr) {
383+
throw std::runtime_error(
384+
"GPUDevice::importExternalTexture(): ImportSharedTextureMemory "
385+
"returned null. Is 'shared-texture-memory-ahardware-buffer' enabled?");
386+
}
387+
388+
// 2. Create the texture. No descriptor: Dawn picks the right format
389+
// (OpaqueYCbCrAndroid for YUV, R8 / RGBA8 / ... for color AHBs).
390+
auto texture = memory.CreateTexture();
391+
if (texture == nullptr) {
392+
throw std::runtime_error(
393+
"GPUDevice::importExternalTexture(): CreateTexture returned null");
394+
}
395+
396+
// 3. Begin access. Vulkan requires us to advertise the incoming VkImage
397+
// layout (UNDEFINED is fine for the first acquisition of an AHB whose
398+
// contents we expect Dawn to read as-is).
399+
wgpu::SharedTextureMemoryBeginAccessDescriptor begin{};
400+
begin.initialized = true;
401+
begin.concurrentRead = false;
402+
wgpu::SharedTextureMemoryVkImageLayoutBeginState beginLayout{};
403+
begin.nextInChain = &beginLayout;
404+
if (!memory.BeginAccess(texture, &begin)) {
405+
throw std::runtime_error(
406+
"GPUDevice::importExternalTexture(): BeginAccess failed");
407+
}
408+
409+
// 4. Build the ExternalTextureDescriptor. Unlike iOS we do *not* split
410+
// planes or pass an explicit YUV→RGB matrix: when the underlying texture
411+
// is OpaqueYCbCrAndroid, Dawn routes sampling through a Vulkan
412+
// SamplerYcbcrConversion that does the conversion implicitly, driven by
413+
// the AHB's own format metadata. This is the "passthrough external
414+
// texture" pattern from Dawn's tests
415+
// (utils::MakePassthroughExternalTexture).
416+
wgpu::ExternalTextureDescriptor extDesc{};
417+
if (!label.empty()) {
418+
extDesc.label = wgpu::StringView(label.c_str(), label.size());
419+
}
420+
extDesc.plane0 = texture.CreateView();
421+
extDesc.cropOrigin = {0, 0};
422+
extDesc.cropSize = {frame.width, frame.height};
423+
extDesc.apparentSize = {frame.width, frame.height};
424+
425+
auto external = _instance.CreateExternalTexture(&extDesc);
426+
if (external == nullptr) {
427+
wgpu::SharedTextureMemoryEndAccessState state{};
428+
(void)memory.EndAccess(texture, &state);
429+
throw std::runtime_error(
430+
"GPUDevice::importExternalTexture(): CreateExternalTexture returned "
431+
"null");
432+
}
433+
366434
return std::make_shared<GPUExternalTexture>(
367435
std::move(external), std::move(memory), std::move(texture),
368436
std::move(descriptor->source), std::move(label));

‎packages/webgpu/cpp/rnwgpu/api/GPUFeatures.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,9 @@ static void convertEnumToJSUnion(wgpu::FeatureName inEnum,
188188
case wgpu::FeatureName::YCbCrVulkanSamplers:
189189
*outUnion = "ycbcr-vulkan-samplers";
190190
break;
191+
case wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture:
192+
*outUnion = "opaque-ycbcr-android-for-external-texture";
193+
break;
191194
case wgpu::FeatureName::ShaderModuleCompilationOptions:
192195
*outUnion = "shader-module-compilation-options";
193196
break;

‎packages/webgpu/cpp/rnwgpu/api/RnFeatures.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,13 @@ inline std::vector<wgpu::FeatureName> rnSharedTextureMemoryBackingFeatures() {
2424
return {wgpu::FeatureName::SharedTextureMemoryIOSurface,
2525
wgpu::FeatureName::SharedFenceMTLSharedEvent};
2626
#elif defined(__ANDROID__)
27+
// OpaqueYCbCrAndroidForExternalTexture is the Vulkan-side equivalent of what
28+
// we get "for free" through IOSurface biplanar textures on Metal: it lets
29+
// CreateExternalTexture wrap an AHB-backed YCbCr texture and have sampling
30+
// route through a SamplerYcbcrConversion implicitly.
2731
return {wgpu::FeatureName::SharedTextureMemoryAHardwareBuffer,
28-
wgpu::FeatureName::SharedFenceSyncFD};
32+
wgpu::FeatureName::SharedFenceSyncFD,
33+
wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture};
2934
#else
3035
return {};
3136
#endif

‎packages/webgpu/cpp/rnwgpu/api/descriptors/Unions.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,8 @@ inline void convertJSUnionToEnum(const std::string &inUnion,
527527
*outEnum = wgpu::FeatureName::StaticSamplers;
528528
} else if (inUnion == "ycbcr-vulkan-samplers") {
529529
*outEnum = wgpu::FeatureName::YCbCrVulkanSamplers;
530+
} else if (inUnion == "opaque-ycbcr-android-for-external-texture") {
531+
*outEnum = wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture;
530532
} else if (inUnion == "shader-module-compilation-options") {
531533
*outEnum = wgpu::FeatureName::ShaderModuleCompilationOptions;
532534
} else if (inUnion == "dawn-load-resolve-texture") {
@@ -718,6 +720,9 @@ inline void convertEnumToJSUnion(wgpu::FeatureName inEnum,
718720
case wgpu::FeatureName::YCbCrVulkanSamplers:
719721
*outUnion = "ycbcr-vulkan-samplers";
720722
break;
723+
case wgpu::FeatureName::OpaqueYCbCrAndroidForExternalTexture:
724+
*outUnion = "opaque-ycbcr-android-for-external-texture";
725+
break;
721726
case wgpu::FeatureName::ShaderModuleCompilationOptions:
722727
*outUnion = "shader-module-compilation-options";
723728
break;

0 commit comments

Comments
 (0)