feat: port image segmentation native code to C++

android/CMakeLists.txt

@@ -11,7 +11,7 @@ string(APPEND CMAKE_CXX_FLAGS " -DRCT_NEW_ARCH_ENABLED")
 
 set(ANDROID_CPP_DIR "${CMAKE_SOURCE_DIR}/src/main/cpp")
 set(COMMON_CPP_DIR "${CMAKE_SOURCE_DIR}/../common")
-set(ET_LIB_DIR "${CMAKE_SOURCE_DIR}/../third-party/android/libs")
-set(ET_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/../third-party/include")
+set(LIBS_DIR "${CMAKE_SOURCE_DIR}/../third-party/android/libs")
+set(INCLUDE_DIR "${CMAKE_SOURCE_DIR}/../third-party/include")
 
 add_subdirectory("${ANDROID_CPP_DIR}")

android/build.gradle

@@ -111,7 +111,7 @@ android {
     targetSdkVersion getExtOrIntegerDefault("targetSdkVersion")
     externalNativeBuild {
         cmake {
-          cppFlags "-O2 -frtti -fexceptions -Wall -fstack-protector-all"
+          cppFlags "-O2 -frtti -fexceptions -Wall -fstack-protector-all -fopenmp -static-openmp"
           abiFilters (*reactNativeArchitectures())
           arguments "-DANDROID_STL=c++_shared",
                     "-DREACT_NATIVE_DIR=${toPlatformFileString(reactNativeRootDir.path)}"
@@ -142,6 +142,7 @@ android {
     sourceCompatibility JavaVersion.VERSION_1_8
     targetCompatibility JavaVersion.VERSION_1_8
   }
+
 }
 
 repositories {
@@ -162,7 +163,7 @@ dependencies {
   implementation 'com.facebook.fbjni:fbjni:0.6.0'
   implementation 'org.opencv:opencv:4.10.0'
   implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
-  implementation(files("../third-party/android/libs/executorch.aar"))
+  implementation(files("../third-party/android/libs/executorch/executorch.aar"))
   implementation 'org.opencv:opencv:4.10.0'
   implementation("com.squareup.okhttp3:okhttp:4.9.2")
 }

android/src/main/cpp/CMakeLists.txt

@@ -13,11 +13,10 @@ target_include_directories(
   PUBLIC
   "${COMMON_CPP_DIR}"
   "${ANDROID_CPP_DIR}"
-  "${ET_INCLUDE_DIR}"
+  "${INCLUDE_DIR}"
   "${REACT_NATIVE_DIR}/ReactCommon"
   "${REACT_NATIVE_DIR}/ReactAndroid/src/main/jni/react/turbomodule"
   "${REACT_NATIVE_DIR}/ReactCommon/callinvoker"
-  "${BUILD_DIR}/generated/source/codegen/jni/react/renderer/components/RnExecutorchSpec"
 )
 
 set(LINK_LIBRARIES
@@ -31,14 +30,42 @@ set(RN_VERSION_LINK_LIBRARIES
   ReactAndroid::reactnative
 )
 
+# Dependencies:
+# ------- Executorch -------
+
 add_library(executorch SHARED IMPORTED)
 
 set_target_properties(executorch PROPERTIES
-                      IMPORTED_LOCATION "${ET_LIB_DIR}/${ANDROID_ABI}/libexecutorch.so")
+                      IMPORTED_LOCATION "${LIBS_DIR}/executorch/${ANDROID_ABI}/libexecutorch.so")
+
+# ------- OpenCV -------
+
+set(OPENCV_LIBS 
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_core.a"
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_features2d.a"
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_highgui.a"
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_imgproc.a"
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_photo.a"
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_video.a"
+)
+
+if(ANDROID_ABI STREQUAL "arm64-v8a")
+  set(OPENCV_THIRD_PARTY_LIBS
+    "${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv_hal.a"
+    "${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv_thread.a"
+    "${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv.a"
+  )
+elseif(ANDROID_ABI STREQUAL "x86_64")
+  set(OPENCV_THIRD_PARTY_LIBS "")
+endif()
+
+# --------------
 
 target_link_libraries(
   react-native-executorch 
   ${LINK_LIBRARIES} 
   ${RN_VERSION_LINK_LIBRARIES}
+  ${OPENCV_LIBS}
+  ${OPENCV_THIRD_PARTY_LIBS}
   executorch
 )

android/src/main/cpp/ETInstallerModule.cpp

@@ -1,8 +1,12 @@
 #include "ETInstallerModule.h"
-#include "RnExecutorchInstaller.h"
 
-namespace rnexecutorch {
+#include <rnexecutorch/RnExecutorchInstaller.h>
+
+#include <jni.h>
+#include <jsi/jsi.h>
 
+namespace rnexecutorch {
+JavaVM *java_machine;
 using namespace facebook::jni;
 
 ETInstallerModule::ETInstallerModule(
@@ -30,6 +34,43 @@ void ETInstallerModule::registerNatives() {
 }
 
 void ETInstallerModule::injectJSIBindings() {
-  RnExecutorchInstaller::injectJSIBindings(jsiRuntime_, jsCallInvoker_);
+  // Grab a function for fetching images via URL from Java
+  auto fetchDataByUrl = [](std::string url) {
+    // Attaching Current Thread to JVM
+
+    JNIEnv *env = nullptr;
+    int getEnvStat = java_machine->GetEnv((void **)&env, JNI_VERSION_1_6);
+    if (getEnvStat == JNI_EDETACHED) {
+      if (java_machine->AttachCurrentThread(&env, nullptr) != 0) {
+        throw std::runtime_error("Failed to attach thread to JVM");
+      }
+    }
+    static jclass cls = javaClassStatic().get();
+    static jmethodID method = env->GetStaticMethodID(
+        cls, "fetchByteDataFromUrl", "(Ljava/lang/String;)[B");
+
+    jstring jUrl = env->NewStringUTF(url.c_str());
+    jbyteArray byteData =
+        (jbyteArray)env->CallStaticObjectMethod(cls, method, jUrl);
+
+    if (env->IsSameObject(byteData, NULL)) {
+      throw std::runtime_error("Error fetching data from a url");
+    }
+
+    int size = env->GetArrayLength(byteData);
+    jbyte *bytes = env->GetByteArrayElements(byteData, JNI_FALSE);
+    std::byte *dataBytePtr = reinterpret_cast<std::byte *>(bytes);
+
+    return std::vector<std::byte>(dataBytePtr, dataBytePtr + size);
+  };
+
+  RnExecutorchInstaller::injectJSIBindings(jsiRuntime_, jsCallInvoker_,
+                                           fetchDataByUrl);
 }
-} // namespace rnexecutorch
+} // namespace rnexecutorch
+
+JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
+  rnexecutorch::java_machine = vm;
+  return facebook::jni::initialize(
+      vm, [] { rnexecutorch::ETInstallerModule::registerNatives(); });
+}

android/src/main/java/com/swmansion/rnexecutorch/ETInstaller.kt

@@ -1,11 +1,14 @@
 package com.swmansion.rnexecutorch
 
 import com.facebook.jni.HybridData
+import com.facebook.proguard.annotations.DoNotStrip
 import com.facebook.react.bridge.ReactApplicationContext
 import com.facebook.react.bridge.ReactMethod
 import com.facebook.react.common.annotations.FrameworkAPI
 import com.facebook.react.module.annotations.ReactModule
 import com.facebook.react.turbomodule.core.CallInvokerHolderImpl
+import java.io.InputStream
+import java.net.URL
 
 @OptIn(FrameworkAPI::class)
 @ReactModule(name = ETInstaller.NAME)
@@ -14,6 +17,25 @@ class ETInstaller(
 ) : NativeETInstallerSpec(reactContext) {
   companion object {
     const val NAME = NativeETInstallerSpec.NAME
+
+    @JvmStatic
+    @DoNotStrip
+    @Throws(Exception::class)
+    fun fetchByteDataFromUrl(source: String): ByteArray? {
+      try {
+        val url = URL(source)
+        val connection = url.openConnection()
+        connection.connect()
+
+        val inputStream: InputStream = connection.getInputStream()
+        val data = inputStream.readBytes()
+        inputStream.close()
+
+        return data
+      } catch (exception: Throwable) {
+        return null
+      }
+    }
   }
 
   private val mHybridData: HybridData
@@ -32,7 +54,7 @@ class ETInstaller(
       val jsCallInvokerHolder = reactContext.jsCallInvokerHolder as CallInvokerHolderImpl
       mHybridData = initHybrid(reactContext.javaScriptContextHolder!!.get(), jsCallInvokerHolder)
     } catch (exception: UnsatisfiedLinkError) {
-      throw RuntimeException("Could not load native module Install", exception)
+      throw RuntimeException("Could not load native module ETInstaller", exception)
     }
   }
 

android/src/main/java/com/swmansion/rnexecutorch/RnExecutorchPackage.kt

@@ -18,8 +18,6 @@ class RnExecutorchPackage : TurboReactPackage() {
       LLM(reactContext)
     } else if (name == ETModule.NAME) {
       ETModule(reactContext)
-    } else if (name == StyleTransfer.NAME) {
-      StyleTransfer(reactContext)
     } else if (name == Classification.NAME) {
       Classification(reactContext)
     } else if (name == ObjectDetection.NAME) {
@@ -30,8 +28,6 @@ class RnExecutorchPackage : TurboReactPackage() {
       OCR(reactContext)
     } else if (name == VerticalOCR.NAME) {
       VerticalOCR(reactContext)
-    } else if (name == ImageSegmentation.NAME) {
-      ImageSegmentation(reactContext)
     } else if (name == ETInstaller.NAME) {
       ETInstaller(reactContext)
     } else if (name == Tokenizer.NAME) {
@@ -66,17 +62,6 @@ class RnExecutorchPackage : TurboReactPackage() {
           true,
         )
 
-      moduleInfos[StyleTransfer.NAME] =
-        ReactModuleInfo(
-          StyleTransfer.NAME,
-          StyleTransfer.NAME,
-          false, // canOverrideExistingModule
-          false, // needsEagerInit
-          true, // hasConstants
-          false, // isCxxModule
-          true,
-        )
-
       moduleInfos[Classification.NAME] =
         ReactModuleInfo(
           Classification.NAME,
@@ -132,17 +117,6 @@ class RnExecutorchPackage : TurboReactPackage() {
           true,
         )
 
-      moduleInfos[ImageSegmentation.NAME] =
-        ReactModuleInfo(
-          ImageSegmentation.NAME,
-          ImageSegmentation.NAME,
-          false, // canOverrideExistingModule
-          false, // needsEagerInit
-          true, // hasConstants
-          false, // isCxxModule
-          true,
-        )
-
       moduleInfos[Tokenizer.NAME] =
         ReactModuleInfo(
           Tokenizer.NAME,