Skip to content

Commit 6093cb0

Browse files
bhamiltoncxbhamiltoncx
committed
feat: Update createImageBitmap() to accept ArrayBuffer or Blob
On Android, React Native 0.82 and later suffers from a serious open issue where `file://` URIs to local assets cannot be loaded with `fetch()`: facebook/react-native#54626 That means it's very difficult to load bundled texture assets to use with `react-native-wgpu` in release builds on Android. Develompent builds work fine, since assets are loaded over HTTP, but release builds directly include assets which resolve to local `file://` URIs like `file:///android_res/drawable/assets_textures_foo.jpg`. Because `react-native-wgpu`'s `createImageBitmap()` currently requires a `Blob`, there's no good workaround for the React Native issue, since the only thing which can create `Blob`s is `fetch()`. (Note: I tried using the workaround from expo/expo#2402 (comment) to create an `XMLHttpRequest` to fetch the `file://` URI, but it fails in React Native 0.82 and later as well.) To resolve this issue, this PR extends `react-native-wgpu`'s `createImageBitmap()` to accept either `Blob` *or* `ArrayBuffer`s containing encoded image bytes (PNG/JPEG/etc.). In specific, this PR: - Upgrades the C++ version to C++20 for `std::span` support to avoid copies - Adds `createImageBitmapFromData(std::span<uint8_t>)` to `PlatformContext` - Refactors the existing `Blob` codepaths to resolve the `Blob` to bytes, then use the `std::span` codepath - Detects `ArrayBuffer` vs `Blob` in `RNWebGPU::createImageBitmap` at runtime - Adds a TypeScript global overload to allow calling `createImageBitmap(ArrayBuffer)` - Updates the `TexturedCube` sample to use the new `ArrayBuffer` codepath I tested this by running the Android and iOS samples in the simulator and on device. I did use Claude Code to help draft this PR, but then edited the code myself and tested it by hand (I promise I'm a human being).
1 parent 0949339 commit 6093cb0

12 files changed

Lines changed: 226 additions & 223 deletions

File tree

‎apps/example/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"pod:install:ios": "pod install --project-directory=ios",
1313
"pod:install:macos": "pod install --project-directory=macos",
1414
"build:android": "cd android && ./gradlew assembleDebug --warning-mode all",
15-
"build:ios": "react-native build-ios --scheme Example --mode Debug --extra-params \"-sdk iphonesimulator CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ GCC_OPTIMIZATION_LEVEL=0 GCC_PRECOMPILE_PREFIX_HEADER=YES ASSETCATALOG_COMPILER_OPTIMIZATION=time DEBUG_INFORMATION_FORMAT=dwarf COMPILER_INDEX_STORE_ENABLE=NO\"",
15+
"build:ios": "react-native build-ios --scheme Example --mode Debug --extra-params \"-sdk iphonesimulator CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ GCC_OPTIMIZATION_LEVEL=0 GCC_PRECOMPILE_PREFIX_HEADER=YES ASSETCATALOG_COMPILER_OPTIMIZATION=time DEBUG_INFORMATION_FORMAT=dwarf COMPILER_INDEX_STORE_ENABLE=NO CLANG_CXX_LANGUAGE_STANDARD=c++20\"",
1616
"build:macos": "react-native build-macos --scheme Example --mode Debug",
1717
"mkdist": "node -e \"require('node:fs').mkdirSync('dist', { recursive: true, mode: 0o755 })\"",
1818
"postinstall": "node -e \"if (process.platform !== 'darwin') { console.log('Skipping iOS pod install on non-macOS environment.'); process.exit(0); } const { execSync } = require('child_process'); execSync('yarn pod:install:ios', { stdio: 'inherit' });\""

‎apps/example/src/Cube/TexturedCube.tsx‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,9 +102,10 @@ export const TexturedCube = () => {
102102

103103
// Fetch the image and upload it into a GPUTexture.
104104
let cubeTexture: GPUTexture;
105-
{
106-
const response = await fetchAsset(require("../assets/Di-3d.png"));
107-
const imageBitmap = await createImageBitmap(await response.blob());
105+
try {
106+
const asset = await fetchAsset(require("../assets/Di-3d.png"));
107+
const arrayBuffer = await asset.arrayBuffer();
108+
const imageBitmap = await createImageBitmap(arrayBuffer);
108109
cubeTexture = device.createTexture({
109110
size: [imageBitmap.width, imageBitmap.height, 1],
110111
format: "rgba8unorm",
@@ -116,8 +117,10 @@ export const TexturedCube = () => {
116117
device.queue.copyExternalImageToTexture(
117118
{ source: imageBitmap },
118119
{ texture: cubeTexture },
119-
[imageBitmap.width, imageBitmap.height],
120+
[imageBitmap.width, imageBitmap.height]
120121
);
122+
} catch (err) {
123+
console.error("Failed to fetch asset", err);
121124
}
122125

123126
// Create a sampler with linear filtering for smooth interpolation.

‎packages/webgpu/android/CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ cmake_minimum_required(VERSION 3.4.1)
22
project(RNWGPU)
33

44
set (CMAKE_VERBOSE_MAKEFILE ON)
5-
set (CMAKE_CXX_STANDARD 17)
5+
set (CMAKE_CXX_STANDARD 20)
6+
set (CMAKE_CXX_STANDARD_REQUIRED True)
67

78
set (PACKAGE_NAME "react-native-wgpu")
89

‎packages/webgpu/android/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ android {
7171
buildConfigField "boolean", "IS_NEW_ARCHITECTURE_ENABLED", "true"
7272
externalNativeBuild {
7373
cmake {
74-
cppFlags "-fexceptions", "-frtti", "-std=c++1y", "-DONANDROID"
74+
cppFlags "-fexceptions", "-frtti", "-DONANDROID"
7575
abiFilters (*reactNativeArchitectures())
7676
arguments '-DANDROID_STL=c++_shared',
7777
"-DNODE_MODULES_DIR=${nodeModules}",

‎packages/webgpu/android/cpp/AndroidPlatformContext.h‎

Lines changed: 92 additions & 139 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,41 @@ class AndroidPlatformContext : public PlatformContext {
2323
private:
2424
jobject _blobModule;
2525

26+
std::vector<uint8_t> resolveBlob(JNIEnv *env, const std::string &blobId,
27+
double offset, double size) {
28+
if (!_blobModule) {
29+
throw std::runtime_error("BlobModule instance is null");
30+
}
31+
32+
jclass blobModuleClass = env->GetObjectClass(_blobModule);
33+
if (!blobModuleClass) {
34+
throw std::runtime_error("Couldn't find BlobModule class");
35+
}
36+
37+
jmethodID resolveMethod = env->GetMethodID(blobModuleClass, "resolve",
38+
"(Ljava/lang/String;II)[B");
39+
if (!resolveMethod) {
40+
throw std::runtime_error("Couldn't find resolve method in BlobModule");
41+
}
42+
43+
jstring jBlobId = env->NewStringUTF(blobId.c_str());
44+
jbyteArray blobData = (jbyteArray)env->CallObjectMethod(
45+
_blobModule, resolveMethod, jBlobId, static_cast<jint>(offset),
46+
static_cast<jint>(size));
47+
env->DeleteLocalRef(jBlobId);
48+
49+
if (!blobData) {
50+
throw std::runtime_error("Couldn't retrieve blob data");
51+
}
52+
53+
jsize len = env->GetArrayLength(blobData);
54+
std::vector<uint8_t> data(len);
55+
env->GetByteArrayRegion(blobData, 0, len,
56+
reinterpret_cast<jbyte *>(data.data()));
57+
env->DeleteLocalRef(blobData);
58+
return data;
59+
}
60+
2661
public:
2762
explicit AndroidPlatformContext(jobject blobModule)
2863
: _blobModule(blobModule) {}
@@ -52,188 +87,106 @@ class AndroidPlatformContext : public PlatformContext {
5287
throw std::runtime_error("Couldn't get JNI environment");
5388
}
5489

55-
// Use the BlobModule instance from _blobModule
56-
if (!_blobModule) {
57-
throw std::runtime_error("BlobModule instance is null");
58-
}
90+
auto data = resolveBlob(env, blobId, offset, size);
91+
return createImageBitmapFromData(data);
92+
}
5993

60-
// Get the resolve method ID
61-
jclass blobModuleClass = env->GetObjectClass(_blobModule);
62-
if (!blobModuleClass) {
63-
throw std::runtime_error("Couldn't find BlobModule class");
64-
}
94+
void createImageBitmapAsync(
95+
std::string blobId, double offset, double size,
96+
std::function<void(ImageData)> onSuccess,
97+
std::function<void(std::string)> onError) override {
98+
std::thread([this, blobId = std::move(blobId), offset, size,
99+
onSuccess = std::move(onSuccess),
100+
onError = std::move(onError)]() {
101+
jni::Environment::ensureCurrentThreadIsAttached();
102+
try {
103+
JNIEnv *env = facebook::jni::Environment::current();
104+
if (!env) {
105+
throw std::runtime_error("Couldn't get JNI environment");
106+
}
107+
auto data = resolveBlob(env, blobId, offset, size);
108+
auto result = createImageBitmapFromData(data);
109+
onSuccess(std::move(result));
110+
} catch (const std::exception &e) {
111+
onError(e.what());
112+
}
113+
}).detach();
114+
}
65115

66-
jmethodID resolveMethod = env->GetMethodID(blobModuleClass, "resolve",
67-
"(Ljava/lang/String;II)[B");
68-
if (!resolveMethod) {
69-
throw std::runtime_error("Couldn't find resolve method in BlobModule");
70-
}
116+
ImageData createImageBitmapFromData(std::span<uint8_t> data) override {
117+
jni::Environment::ensureCurrentThreadIsAttached();
71118

72-
// Resolve the blob data
73-
jstring jBlobId = env->NewStringUTF(blobId.c_str());
74-
jbyteArray blobData = (jbyteArray)env->CallObjectMethod(
75-
_blobModule, resolveMethod, jBlobId, static_cast<jint>(offset),
76-
static_cast<jint>(size));
77-
env->DeleteLocalRef(jBlobId);
119+
JNIEnv *env = facebook::jni::Environment::current();
120+
if (!env) {
121+
throw std::runtime_error("Couldn't get JNI environment");
122+
}
78123

79-
if (!blobData) {
80-
throw std::runtime_error("Couldn't retrieve blob data");
124+
// Create jbyteArray from the raw bytes
125+
jbyteArray byteArray = env->NewByteArray(static_cast<jsize>(data.size()));
126+
if (!byteArray) {
127+
throw std::runtime_error("Couldn't allocate byte array");
81128
}
129+
env->SetByteArrayRegion(byteArray, 0, static_cast<jsize>(data.size()),
130+
reinterpret_cast<const jbyte *>(data.data()));
82131

83-
// Create a Bitmap from the blob data
132+
// Decode via BitmapFactory
84133
jclass bitmapFactoryClass =
85134
env->FindClass("android/graphics/BitmapFactory");
86135
jmethodID decodeByteArrayMethod =
87136
env->GetStaticMethodID(bitmapFactoryClass, "decodeByteArray",
88137
"([BII)Landroid/graphics/Bitmap;");
89-
jint blobLength = env->GetArrayLength(blobData);
138+
jint length = static_cast<jint>(data.size());
90139
jobject bitmap = env->CallStaticObjectMethod(
91-
bitmapFactoryClass, decodeByteArrayMethod, blobData, 0, blobLength);
140+
bitmapFactoryClass, decodeByteArrayMethod, byteArray, 0, length);
92141

93142
if (!bitmap) {
94-
env->DeleteLocalRef(blobData);
143+
env->DeleteLocalRef(byteArray);
95144
throw std::runtime_error("Couldn't decode image");
96145
}
97146

98-
// Get bitmap info
99147
AndroidBitmapInfo bitmapInfo;
100148
if (AndroidBitmap_getInfo(env, bitmap, &bitmapInfo) !=
101149
ANDROID_BITMAP_RESULT_SUCCESS) {
102-
env->DeleteLocalRef(blobData);
150+
env->DeleteLocalRef(byteArray);
103151
env->DeleteLocalRef(bitmap);
104152
throw std::runtime_error("Couldn't get bitmap info");
105153
}
106154

107-
// Lock the bitmap pixels
108155
void *bitmapPixels;
109156
if (AndroidBitmap_lockPixels(env, bitmap, &bitmapPixels) !=
110157
ANDROID_BITMAP_RESULT_SUCCESS) {
111-
env->DeleteLocalRef(blobData);
158+
env->DeleteLocalRef(byteArray);
112159
env->DeleteLocalRef(bitmap);
113160
throw std::runtime_error("Couldn't lock bitmap pixels");
114161
}
115162

116-
// Copy the bitmap data
117-
std::vector<uint8_t> imageData(bitmapInfo.height * bitmapInfo.stride);
118-
memcpy(imageData.data(), bitmapPixels, imageData.size());
163+
ImageData result;
164+
result.width = static_cast<int>(bitmapInfo.width);
165+
result.height = static_cast<int>(bitmapInfo.height);
166+
result.data.resize(bitmapInfo.height * bitmapInfo.stride);
167+
memcpy(result.data.data(), bitmapPixels, result.data.size());
119168

120-
// Unlock the bitmap pixels
121169
AndroidBitmap_unlockPixels(env, bitmap);
122170

123-
// Clean up JNI references
124-
env->DeleteLocalRef(blobData);
171+
env->DeleteLocalRef(byteArray);
125172
env->DeleteLocalRef(bitmap);
126173

127-
ImageData result;
128-
result.width = static_cast<int>(bitmapInfo.width);
129-
result.height = static_cast<int>(bitmapInfo.height);
130-
result.data = imageData;
131174
return result;
132175
}
133176

134-
void createImageBitmapAsync(
135-
std::string blobId, double offset, double size,
136-
std::function<void(ImageData)> onSuccess,
177+
void createImageBitmapFromDataAsync(
178+
std::span<uint8_t> data, std::function<void(ImageData)> onSuccess,
137179
std::function<void(std::string)> onError) override {
138-
// Capture blobModule for the background thread
139-
jobject blobModule = _blobModule;
140-
141-
// Dispatch to a background thread
142-
std::thread([blobModule, blobId = std::move(blobId), offset, size,
180+
std::thread([this, ownedData = std::vector<uint8_t>(data.begin(), data.end()),
143181
onSuccess = std::move(onSuccess),
144-
onError = std::move(onError)]() {
182+
onError = std::move(onError)]() mutable {
145183
jni::Environment::ensureCurrentThreadIsAttached();
146-
147-
JNIEnv *env = facebook::jni::Environment::current();
148-
if (!env) {
149-
onError("Couldn't get JNI environment");
150-
return;
151-
}
152-
153-
if (!blobModule) {
154-
onError("BlobModule instance is null");
155-
return;
156-
}
157-
158-
// Get the resolve method ID
159-
jclass blobModuleClass = env->GetObjectClass(blobModule);
160-
if (!blobModuleClass) {
161-
onError("Couldn't find BlobModule class");
162-
return;
163-
}
164-
165-
jmethodID resolveMethod = env->GetMethodID(blobModuleClass, "resolve",
166-
"(Ljava/lang/String;II)[B");
167-
if (!resolveMethod) {
168-
onError("Couldn't find resolve method in BlobModule");
169-
return;
170-
}
171-
172-
// Resolve the blob data
173-
jstring jBlobId = env->NewStringUTF(blobId.c_str());
174-
jbyteArray blobData = (jbyteArray)env->CallObjectMethod(
175-
blobModule, resolveMethod, jBlobId, static_cast<jint>(offset),
176-
static_cast<jint>(size));
177-
env->DeleteLocalRef(jBlobId);
178-
179-
if (!blobData) {
180-
onError("Couldn't retrieve blob data");
181-
return;
184+
try {
185+
auto result = createImageBitmapFromData(ownedData);
186+
onSuccess(std::move(result));
187+
} catch (const std::exception &e) {
188+
onError(e.what());
182189
}
183-
184-
// Create a Bitmap from the blob data
185-
jclass bitmapFactoryClass =
186-
env->FindClass("android/graphics/BitmapFactory");
187-
jmethodID decodeByteArrayMethod =
188-
env->GetStaticMethodID(bitmapFactoryClass, "decodeByteArray",
189-
"([BII)Landroid/graphics/Bitmap;");
190-
jint blobLength = env->GetArrayLength(blobData);
191-
jobject bitmap = env->CallStaticObjectMethod(
192-
bitmapFactoryClass, decodeByteArrayMethod, blobData, 0, blobLength);
193-
194-
if (!bitmap) {
195-
env->DeleteLocalRef(blobData);
196-
onError("Couldn't decode image");
197-
return;
198-
}
199-
200-
// Get bitmap info
201-
AndroidBitmapInfo bitmapInfo;
202-
if (AndroidBitmap_getInfo(env, bitmap, &bitmapInfo) !=
203-
ANDROID_BITMAP_RESULT_SUCCESS) {
204-
env->DeleteLocalRef(blobData);
205-
env->DeleteLocalRef(bitmap);
206-
onError("Couldn't get bitmap info");
207-
return;
208-
}
209-
210-
// Lock the bitmap pixels
211-
void *bitmapPixels;
212-
if (AndroidBitmap_lockPixels(env, bitmap, &bitmapPixels) !=
213-
ANDROID_BITMAP_RESULT_SUCCESS) {
214-
env->DeleteLocalRef(blobData);
215-
env->DeleteLocalRef(bitmap);
216-
onError("Couldn't lock bitmap pixels");
217-
return;
218-
}
219-
220-
// Copy the bitmap data
221-
std::vector<uint8_t> imageData(bitmapInfo.height * bitmapInfo.stride);
222-
memcpy(imageData.data(), bitmapPixels, imageData.size());
223-
224-
// Unlock the bitmap pixels
225-
AndroidBitmap_unlockPixels(env, bitmap);
226-
227-
// Clean up JNI references
228-
env->DeleteLocalRef(blobData);
229-
env->DeleteLocalRef(bitmap);
230-
231-
ImageData result;
232-
result.width = static_cast<int>(bitmapInfo.width);
233-
result.height = static_cast<int>(bitmapInfo.height);
234-
result.data = std::move(imageData);
235-
236-
onSuccess(std::move(result));
237190
}).detach();
238191
}
239192
};

‎packages/webgpu/apple/ApplePlatformContext.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,12 @@ class ApplePlatformContext : public PlatformContext {
2020
std::string blobId, double offset, double size,
2121
std::function<void(ImageData)> onSuccess,
2222
std::function<void(std::string)> onError) override;
23+
24+
ImageData createImageBitmapFromData(std::span<uint8_t> data) override;
25+
26+
void createImageBitmapFromDataAsync(
27+
std::span<uint8_t> data, std::function<void(ImageData)> onSuccess,
28+
std::function<void(std::string)> onError) override;
2329
};
2430

2531
} // namespace rnwgpu

0 commit comments

Comments
 (0)