Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,15 @@ docs/docs/06-api-reference/
# integration test model assets
packages/react-native-executorch/common/rnexecutorch/tests/integration/assets/models/

# release artifact staging dir (produced by scripts/package-release-artifacts.sh)
packages/react-native-executorch/dist-artifacts/

# on-demand native libs (downloaded at postinstall time, not committed)
packages/react-native-executorch/third-party/android/libs/
packages/react-native-executorch/third-party/ios/ExecutorchLib.xcframework/
packages/react-native-executorch/third-party/ios/libs/
packages/react-native-executorch/rne-build-config.json

# custom
*.tgz
Makefile
Expand Down
4 changes: 4 additions & 0 deletions packages/react-native-executorch/android/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ set(LIBS_DIR "${CMAKE_SOURCE_DIR}/../third-party/android/libs")
set(TOKENIZERS_DIR "${CMAKE_SOURCE_DIR}/../third-party/include/executorch/extension/llm/tokenizers/include")
set(INCLUDE_DIR "${CMAKE_SOURCE_DIR}/../third-party/include")

# Optional feature flags — driven by user config in package.json, passed via gradle cmake arguments
option(RNE_ENABLE_OPENCV "Enable OpenCV-dependent computer vision features" ON)
option(RNE_ENABLE_PHONEMIZER "Enable Phonemizer-dependent TTS features" ON)

# Treat third-party headers as system headers to suppress deprecation warnings
include_directories(SYSTEM "${INCLUDE_DIR}")

Expand Down
21 changes: 20 additions & 1 deletion packages/react-native-executorch/android/build.gradle
Original file line number Diff line number Diff line change
@@ -1,5 +1,22 @@
import org.apache.tools.ant.taskdefs.condition.Os

// Read the generated build config written by the postinstall script.
// Falls back to enabling everything if the file doesn't exist (e.g. during CI
// when libs are pre-cached and the postinstall script skipped writing config).
def getRneBuildConfig() {
def configFile = new File("${project.projectDir}/../rne-build-config.json")
if (configFile.exists()) {
try {
return new groovy.json.JsonSlurper().parse(configFile)
} catch (e) {
logger.warn("[RnExecutorch] Failed to parse rne-build-config.json: ${e.message}. Defaulting to all features enabled.")
}
}
return [enableOpencv: true, enablePhonemizer: true]
}

def rneBuildConfig = getRneBuildConfig()

buildscript {
ext {
agp_version = '8.4.2'
Expand Down Expand Up @@ -122,7 +139,9 @@ android {
"-DREACT_NATIVE_DIR=${toPlatformFileString(reactNativeRootDir.path)}",
"-DBUILD_DIR=${project.buildDir}",
"-DANDROID_TOOLCHAIN=clang",
"-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON"
"-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON",
"-DRNE_ENABLE_OPENCV=${rneBuildConfig.enableOpencv ? 'ON' : 'OFF'}",
"-DRNE_ENABLE_PHONEMIZER=${rneBuildConfig.enablePhonemizer ? 'ON' : 'OFF'}"
}
}
}
Expand Down
135 changes: 94 additions & 41 deletions packages/react-native-executorch/android/src/main/cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,12 +1,74 @@
cmake_minimum_required(VERSION 3.13)

file(GLOB_RECURSE ANDROID_CPP_SOURCES CONFIGURE_DEPENDS "${ANDROID_CPP_DIR}/*.cpp")
file(GLOB_RECURSE COMMON_CPP_SOURCES CONFIGURE_DEPENDS "${COMMON_CPP_DIR}/*.cpp")
file(GLOB_RECURSE COMMON_C_SOURCES CONFIGURE_DEPENDS "${COMMON_CPP_DIR}/*.c")

# --- Source separation ---
# Glob all common sources, then separate opencv-dependent and phonemizer-dependent
# files so they can be conditionally included based on feature flags.

file(GLOB_RECURSE ALL_COMMON_CPP_SOURCES CONFIGURE_DEPENDS "${COMMON_CPP_DIR}/*.cpp")
file(GLOB_RECURSE ALL_COMMON_C_SOURCES CONFIGURE_DEPENDS "${COMMON_CPP_DIR}/*.c")

# Exclude test sources unconditionally
file(GLOB_RECURSE TEST_CPP_SOURCES "${COMMON_CPP_DIR}/rnexecutorch/tests/*.cpp")
list(REMOVE_ITEM COMMON_CPP_SOURCES ${TEST_CPP_SOURCES})
list(REMOVE_ITEM ALL_COMMON_CPP_SOURCES ${TEST_CPP_SOURCES})

# OpenCV-dependent sources: CV models + frame utilities + image processing
file(GLOB_RECURSE OPENCV_CPP_SOURCES CONFIGURE_DEPENDS
"${COMMON_CPP_DIR}/rnexecutorch/models/classification/*.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/models/object_detection/*.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/models/semantic_segmentation/*.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/models/instance_segmentation/*.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/models/style_transfer/*.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/models/ocr/*.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/models/vertical_ocr/*.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/models/embeddings/image/*.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/models/text_to_image/*.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/models/VisionModel.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/data_processing/ImageProcessing.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/utils/FrameExtractor.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/utils/FrameProcessor.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/utils/FrameTransform.cpp"
"${COMMON_CPP_DIR}/rnexecutorch/utils/computer_vision/*.cpp"
"${COMMON_CPP_DIR}/runner/encoders/vision_encoder.cpp"
"${COMMON_CPP_DIR}/runner/multimodal_prefiller.cpp"
"${COMMON_CPP_DIR}/runner/multimodal_runner.cpp"
)

# Phonemizer-dependent sources: Kokoro TTS (only user of phonemis)
file(GLOB_RECURSE PHONEMIZER_CPP_SOURCES CONFIGURE_DEPENDS
"${COMMON_CPP_DIR}/rnexecutorch/models/text_to_speech/*.cpp"
)

# Core = everything minus optional sources
set(CORE_COMMON_CPP_SOURCES ${ALL_COMMON_CPP_SOURCES})
list(REMOVE_ITEM CORE_COMMON_CPP_SOURCES ${OPENCV_CPP_SOURCES} ${PHONEMIZER_CPP_SOURCES})

# Build final source list
set(ENABLED_COMMON_SOURCES ${CORE_COMMON_CPP_SOURCES})

add_library(react-native-executorch SHARED ${ANDROID_CPP_SOURCES} ${COMMON_CPP_SOURCES} ${COMMON_C_SOURCES})
if(RNE_ENABLE_OPENCV)
list(APPEND ENABLED_COMMON_SOURCES ${OPENCV_CPP_SOURCES})
endif()

if(RNE_ENABLE_PHONEMIZER)
list(APPEND ENABLED_COMMON_SOURCES ${PHONEMIZER_CPP_SOURCES})
endif()

add_library(react-native-executorch SHARED
${ANDROID_CPP_SOURCES}
${ENABLED_COMMON_SOURCES}
${ALL_COMMON_C_SOURCES}
)

# Propagate feature flags as preprocessor defines so C++ code can guard includes
if(RNE_ENABLE_OPENCV)
target_compile_definitions(react-native-executorch PRIVATE RNE_ENABLE_OPENCV)
endif()

if(RNE_ENABLE_PHONEMIZER)
target_compile_definitions(react-native-executorch PRIVATE RNE_ENABLE_PHONEMIZER)
endif()

find_package(ReactAndroid REQUIRED CONFIG)
find_package(fbjni REQUIRED CONFIG)
Expand Down Expand Up @@ -34,63 +96,55 @@ set(RN_VERSION_LINK_LIBRARIES
ReactAndroid::reactnative
)

# Dependencies:

# ------- Executorch -------
# ------- Executorch (always required) -------

add_library(executorch SHARED IMPORTED)

set_target_properties(executorch PROPERTIES
IMPORTED_LOCATION "${LIBS_DIR}/executorch/${ANDROID_ABI}/libexecutorch.so")


if(ANDROID_ABI STREQUAL "arm64-v8a")
target_compile_definitions(react-native-executorch PRIVATE ARCH_ARM64)

# ------- pthreadpool -------
add_library(pthreadpool SHARED IMPORTED)

set_target_properties(pthreadpool PROPERTIES
IMPORTED_LOCATION "${LIBS_DIR}/pthreadpool/${ANDROID_ABI}/libpthreadpool.so")

# ------- cpuinfo -------
add_library(cpuinfo SHARED IMPORTED)

set_target_properties(cpuinfo PROPERTIES
IMPORTED_LOCATION "${LIBS_DIR}/cpuinfo/${ANDROID_ABI}/libcpuinfo.so")
set(EXECUTORCH_LIBS
"pthreadpool"
"cpuinfo"
)

set(EXECUTORCH_LIBS "pthreadpool" "cpuinfo")
endif()

# ------- OpenCV -------
# ------- OpenCV (optional) -------

set(OPENCV_LIBS
"${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_core.a"
"${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_features2d.a"
"${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_highgui.a"
"${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_imgproc.a"
"${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_photo.a"
"${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_video.a"
)

if(ANDROID_ABI STREQUAL "arm64-v8a")
set(OPENCV_THIRD_PARTY_LIBS
"${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv_hal.a"
"${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv_thread.a"
"${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv.a"
if(RNE_ENABLE_OPENCV)
set(OPENCV_LINK_LIBS
"${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_core.a"
"${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_features2d.a"
"${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_highgui.a"
"${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_imgproc.a"
"${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_photo.a"
"${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_video.a"
)
elseif(ANDROID_ABI STREQUAL "x86_64")
set(OPENCV_THIRD_PARTY_LIBS "")
endif()

if(ANDROID_ABI STREQUAL "arm64-v8a")
list(APPEND OPENCV_LINK_LIBS
"${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv_hal.a"
"${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv_thread.a"
"${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv.a"
)
endif()
endif()

# ------- phonemis -------
# ------- Phonemizer (optional) -------

set(PHONEMIS_LIBS
"${LIBS_DIR}/phonemis/${ANDROID_ABI}/libphonemis.a"
)
if(RNE_ENABLE_PHONEMIZER)
set(PHONEMIZER_LINK_LIBS
"${LIBS_DIR}/phonemis/${ANDROID_ABI}/libphonemis.a"
)
endif()

# --------------

Expand All @@ -100,9 +154,8 @@ target_link_libraries(
react-native-executorch
${LINK_LIBRARIES}
${RN_VERSION_LINK_LIBRARIES}
${OPENCV_LIBS}
${OPENCV_THIRD_PARTY_LIBS}
${PHONEMIS_LIBS}
${OPENCV_LINK_LIBS}
${PHONEMIZER_LINK_LIBS}
executorch
${EXECUTORCH_LIBS}
z
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,28 @@

#include <rnexecutorch/TokenizerModule.h>
#include <rnexecutorch/host_objects/JsiConversions.h>
#include <rnexecutorch/models/embeddings/text/TextEmbeddings.h>
#include <rnexecutorch/models/llm/LLM.h>
#include <rnexecutorch/models/speech_to_text/SpeechToText.h>
#include <rnexecutorch/models/voice_activity_detection/VoiceActivityDetection.h>
#include <rnexecutorch/threads/GlobalThreadPool.h>
#include <rnexecutorch/threads/utils/ThreadUtils.h>

#ifdef RNE_ENABLE_OPENCV
#include <rnexecutorch/models/classification/Classification.h>
#include <rnexecutorch/models/embeddings/image/ImageEmbeddings.h>
#include <rnexecutorch/models/embeddings/text/TextEmbeddings.h>
#include <rnexecutorch/models/instance_segmentation/BaseInstanceSegmentation.h>
#include <rnexecutorch/models/llm/LLM.h>
#include <rnexecutorch/models/object_detection/ObjectDetection.h>
#include <rnexecutorch/models/ocr/OCR.h>
#include <rnexecutorch/models/semantic_segmentation/BaseSemanticSegmentation.h>
#include <rnexecutorch/models/speech_to_text/SpeechToText.h>
#include <rnexecutorch/models/style_transfer/StyleTransfer.h>
#include <rnexecutorch/models/text_to_image/TextToImage.h>
#include <rnexecutorch/models/text_to_speech/TextToSpeech.h>
#include <rnexecutorch/models/vertical_ocr/VerticalOCR.h>
#include <rnexecutorch/models/voice_activity_detection/VoiceActivityDetection.h>
#include <rnexecutorch/threads/GlobalThreadPool.h>
#include <rnexecutorch/threads/utils/ThreadUtils.h>
#endif

#ifdef RNE_ENABLE_PHONEMIZER
#include <rnexecutorch/models/text_to_speech/TextToSpeech.h>
#endif

#if defined(__ANDROID__) && defined(__aarch64__)
#include <executorch/extension/threadpool/cpuinfo_utils.h>
Expand All @@ -40,6 +46,7 @@ void RnExecutorchInstaller::injectJSIBindings(
jsiRuntime->global().setProperty(*jsiRuntime, "__rne_isEmulator",
jsi::Value(isEmulator));

#ifdef RNE_ENABLE_OPENCV
jsiRuntime->global().setProperty(
*jsiRuntime, "loadStyleTransfer",
RnExecutorchInstaller::loadModel<models::style_transfer::StyleTransfer>(
Expand Down Expand Up @@ -72,6 +79,7 @@ void RnExecutorchInstaller::injectJSIBindings(
RnExecutorchInstaller::loadModel<
models::object_detection::ObjectDetection>(jsiRuntime, jsCallInvoker,
"loadObjectDetection"));
#endif // RNE_ENABLE_OPENCV

jsiRuntime->global().setProperty(
*jsiRuntime, "loadExecutorchModule",
Expand All @@ -83,10 +91,12 @@ void RnExecutorchInstaller::injectJSIBindings(
RnExecutorchInstaller::loadModel<TokenizerModule>(
jsiRuntime, jsCallInvoker, "loadTokenizerModule"));

#ifdef RNE_ENABLE_OPENCV
jsiRuntime->global().setProperty(
*jsiRuntime, "loadImageEmbeddings",
RnExecutorchInstaller::loadModel<models::embeddings::ImageEmbeddings>(
jsiRuntime, jsCallInvoker, "loadImageEmbeddings"));
#endif // RNE_ENABLE_OPENCV

jsiRuntime->global().setProperty(
*jsiRuntime, "loadTextEmbeddings",
Expand All @@ -98,6 +108,7 @@ void RnExecutorchInstaller::injectJSIBindings(
RnExecutorchInstaller::loadModel<models::llm::LLM>(
jsiRuntime, jsCallInvoker, "loadLLM"));

#ifdef RNE_ENABLE_OPENCV
jsiRuntime->global().setProperty(
*jsiRuntime, "loadOCR",
RnExecutorchInstaller::loadModel<models::ocr::OCR>(
Expand All @@ -107,16 +118,19 @@ void RnExecutorchInstaller::injectJSIBindings(
*jsiRuntime, "loadVerticalOCR",
RnExecutorchInstaller::loadModel<models::ocr::VerticalOCR>(
jsiRuntime, jsCallInvoker, "loadVerticalOCR"));
#endif // RNE_ENABLE_OPENCV

jsiRuntime->global().setProperty(
*jsiRuntime, "loadSpeechToText",
RnExecutorchInstaller::loadModel<models::speech_to_text::SpeechToText>(
jsiRuntime, jsCallInvoker, "loadSpeechToText"));

#ifdef RNE_ENABLE_PHONEMIZER
jsiRuntime->global().setProperty(
*jsiRuntime, "loadTextToSpeechKokoro",
RnExecutorchInstaller::loadModel<models::text_to_speech::kokoro::Kokoro>(
jsiRuntime, jsCallInvoker, "loadTextToSpeechKokoro"));
#endif // RNE_ENABLE_PHONEMIZER

jsiRuntime->global().setProperty(
*jsiRuntime, "loadVAD",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
#include <rnexecutorch/Error.h>
#include <rnexecutorch/Log.h>
#include <rnexecutorch/threads/GlobalThreadPool.h>
#include <runner/text_runner.h>
#ifdef RNE_ENABLE_OPENCV
#include <runner/encoders/vision_encoder.h>
#include <runner/multimodal_runner.h>
#include <runner/text_runner.h>
#endif

namespace rnexecutorch::models::llm {
namespace llm = ::executorch::extension::llm;
Expand All @@ -22,10 +24,8 @@ LLM::LLM(const std::string &modelSource, const std::string &tokenizerSource,
std::shared_ptr<react::CallInvoker> callInvoker)
: BaseModel(modelSource, callInvoker, Module::LoadMode::File) {

if (capabilities.empty()) {
runner_ =
std::make_unique<llm::TextRunner>(std::move(module_), tokenizerSource);
} else {
#ifdef RNE_ENABLE_OPENCV
if (!capabilities.empty()) {
std::map<llm::MultimodalType, std::unique_ptr<llm::IEncoder>> encoders;
for (const auto &cap : capabilities) {
if (cap == "vision") {
Expand All @@ -35,7 +35,13 @@ LLM::LLM(const std::string &modelSource, const std::string &tokenizerSource,
}
runner_ = std::make_unique<llm::MultimodalRunner>(
std::move(module_), tokenizerSource, std::move(encoders));
} else {
#endif
runner_ =
std::make_unique<llm::TextRunner>(std::move(module_), tokenizerSource);
#ifdef RNE_ENABLE_OPENCV
}
#endif

auto loadResult = runner_->load();
if (loadResult != Error::Ok) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
#include <executorch/extension/threadpool/cpuinfo_utils.h>
#include <memory>
#include <mutex>
#ifdef RNE_ENABLE_OPENCV
#include <opencv2/opencv.hpp>
#endif
#include <optional>
#include <rnexecutorch/Log.h>
#include <rnexecutorch/threads/HighPerformanceThreadPool.h>
Expand Down Expand Up @@ -41,7 +43,9 @@ class GlobalThreadPool {
config);
// Disable OpenCV's internal threading to prevent it from overriding our
// thread pool configuration, which would cause degraded performance
#ifdef RNE_ENABLE_OPENCV
cv::setNumThreads(0);
#endif
});
}

Expand Down
Loading
Loading