feat: port style transfer implementation to C++ (#229)

Port native implementation of style transfer to C++ #227. This is a part
of a larger effort to merge native code from Kotlin and Objective C to a
single implementation in C++.

# Changes by sections
## Typescript
Old style transfer modules have been replaced by new, stateful
implementation. For this purpose non-static versions of useModule and
BaseModule have been introduced.
## Objective C & Kotlin
Old implementations of style transfer have been removed. Added methods
for handling fetching data via https that are passed on as function
objects to C++.
## C++
- Ported image processing capabilities equivalent to the ObjC/Kotlin
implementations.
- `ModelHostObject` serves as an automatic interface between model
implementations and JSI. This is done by template metaprogramming;
defining methods in `JsiConversions.h` for types used by the model is
necessary for it to work.
- A factory method for loading a style transfer model is registered in
`RnExecutorchInstaller`.

# Code attribution
- `common/rnexecutorch/jsi` originates from react-native-audio-api
- `base64.*` is used for converting from base64 due to
https://renenyffenegger.ch/notes/development/Base64/Encoding-and-decoding-base-64-with-cpp/
- `ada` is a header-only library for parsing urls due to
https://github.com/ada-url/ada
# Dependencies introduced/updated
- headers for Executorch 0.6 have been updated
- OpenCV 4.11.0 dependency is introduced for Android/C++. For iOS the
version is bumped via Cocoapods to 4.11.0.
- C10 headers are the dependency of OpenCV.

# Known issues
- Exceptions in native C++ do not result in promise getting rejected
when forwarding yet.
- Garbage collection can malfunction for host objects. C++ host objects
need to notify JS runtime about their size via external memory pressure
for the garbage collector to know to free them. This is not yet done.

Author: JakubGonera

android/CMakeLists.txt

@@ -11,7 +11,7 @@ string(APPEND CMAKE_CXX_FLAGS " -DRCT_NEW_ARCH_ENABLED")
 
 set(ANDROID_CPP_DIR "${CMAKE_SOURCE_DIR}/src/main/cpp")
 set(COMMON_CPP_DIR "${CMAKE_SOURCE_DIR}/../common")
-set(ET_LIB_DIR "${CMAKE_SOURCE_DIR}/../third-party/android/libs")
-set(ET_INCLUDE_DIR "${CMAKE_SOURCE_DIR}/../third-party/include")
+set(LIBS_DIR "${CMAKE_SOURCE_DIR}/../third-party/android/libs")
+set(INCLUDE_DIR "${CMAKE_SOURCE_DIR}/../third-party/include")
 
 add_subdirectory("${ANDROID_CPP_DIR}")

android/build.gradle

@@ -111,7 +111,7 @@ android {
     targetSdkVersion getExtOrIntegerDefault("targetSdkVersion")
     externalNativeBuild {
         cmake {
-          cppFlags "-O2 -frtti -fexceptions -Wall -fstack-protector-all"
+          cppFlags "-O2 -frtti -fexceptions -Wall -fstack-protector-all -fopenmp -static-openmp"
           abiFilters (*reactNativeArchitectures())
           arguments "-DANDROID_STL=c++_shared",
                     "-DREACT_NATIVE_DIR=${toPlatformFileString(reactNativeRootDir.path)}"
@@ -162,7 +162,7 @@ dependencies {
   implementation 'com.facebook.fbjni:fbjni:0.6.0'
   implementation 'org.opencv:opencv:4.10.0'
   implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
-  implementation(files("../third-party/android/libs/executorch.aar"))
+  implementation(files("../third-party/android/libs/executorch/executorch.aar"))
   implementation 'org.opencv:opencv:4.10.0'
   implementation("com.squareup.okhttp3:okhttp:4.9.2")
 }

android/src/main/cpp/CMakeLists.txt

@@ -13,11 +13,10 @@ target_include_directories(
   PUBLIC
   "${COMMON_CPP_DIR}"
   "${ANDROID_CPP_DIR}"
-  "${ET_INCLUDE_DIR}"
+  "${INCLUDE_DIR}"
   "${REACT_NATIVE_DIR}/ReactCommon"
   "${REACT_NATIVE_DIR}/ReactAndroid/src/main/jni/react/turbomodule"
   "${REACT_NATIVE_DIR}/ReactCommon/callinvoker"
-  "${BUILD_DIR}/generated/source/codegen/jni/react/renderer/components/RnExecutorchSpec"
 )
 
 set(LINK_LIBRARIES
@@ -31,14 +30,42 @@ set(RN_VERSION_LINK_LIBRARIES
   ReactAndroid::reactnative
 )
 
+# Dependencies:
+# ------- Executorch -------
+
 add_library(executorch SHARED IMPORTED)
 
 set_target_properties(executorch PROPERTIES
-                      IMPORTED_LOCATION "${ET_LIB_DIR}/${ANDROID_ABI}/libexecutorch.so")
+                      IMPORTED_LOCATION "${LIBS_DIR}/executorch/${ANDROID_ABI}/libexecutorch.so")
+
+# ------- OpenCV -------
+
+set(OPENCV_LIBS 
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_core.a"
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_features2d.a"
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_highgui.a"
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_imgproc.a"
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_photo.a"
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_video.a"
+)
+
+if(ANDROID_ABI STREQUAL "arm64-v8a")
+  set(OPENCV_THIRD_PARTY_LIBS
+    "${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv_hal.a"
+    "${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv_thread.a"
+    "${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv.a"
+  )
+elseif(ANDROID_ABI STREQUAL "x86_64")
+  set(OPENCV_THIRD_PARTY_LIBS "")
+endif()
+
+# --------------
 
 target_link_libraries(
   react-native-executorch 
   ${LINK_LIBRARIES} 
   ${RN_VERSION_LINK_LIBRARIES}
+  ${OPENCV_LIBS}
+  ${OPENCV_THIRD_PARTY_LIBS}
   executorch
 )

android/src/main/cpp/ETInstallerModule.cpp

@@ -1,8 +1,12 @@
 #include "ETInstallerModule.h"
-#include "RnExecutorchInstaller.h"
 
-namespace rnexecutorch {
+#include <rnexecutorch/RnExecutorchInstaller.h>
+
+#include <jni.h>
+#include <jsi/jsi.h>
 
+namespace rnexecutorch {
+JavaVM *java_machine;
 using namespace facebook::jni;
 
 ETInstallerModule::ETInstallerModule(
@@ -30,6 +34,39 @@ void ETInstallerModule::registerNatives() {
 }
 
 void ETInstallerModule::injectJSIBindings() {
-  RnExecutorchInstaller::injectJSIBindings(jsiRuntime_, jsCallInvoker_);
+  // Grab a function for fetching images via URL from Java
+  auto fetchDataByUrl = [](std::string url) {
+    // Attaching Current Thread to JVM
+
+    JNIEnv *env = nullptr;
+    int getEnvStat = java_machine->GetEnv((void **)&env, JNI_VERSION_1_6);
+    if (getEnvStat == JNI_EDETACHED) {
+      if (java_machine->AttachCurrentThread(&env, nullptr) != 0) {
+        throw std::runtime_error("Failed to attach thread to JVM");
+      }
+    }
+    static jclass cls = javaClassStatic().get();
+    static jmethodID method = env->GetStaticMethodID(
+        cls, "fetchByteDataFromUrl", "(Ljava/lang/String;)[B");
+
+    jstring jUrl = env->NewStringUTF(url.c_str());
+    jbyteArray byteData =
+        (jbyteArray)env->CallStaticObjectMethod(cls, method, jUrl);
+
+    int size = env->GetArrayLength(byteData);
+    jbyte *bytes = env->GetByteArrayElements(byteData, JNI_FALSE);
+    std::byte *dataBytePtr = reinterpret_cast<std::byte *>(bytes);
+
+    return std::vector<std::byte>(dataBytePtr, dataBytePtr + size);
+  };
+
+  RnExecutorchInstaller::injectJSIBindings(jsiRuntime_, jsCallInvoker_,
+                                           fetchDataByUrl);
 }
-} // namespace rnexecutorch
+} // namespace rnexecutorch
+
+JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
+  rnexecutorch::java_machine = vm;
+  return facebook::jni::initialize(
+      vm, [] { rnexecutorch::ETInstallerModule::registerNatives(); });
+}

android/src/main/cpp/OnLoad.cpp

@@ -1,11 +1,14 @@
 package com.swmansion.rnexecutorch
 
 import com.facebook.jni.HybridData
+import com.facebook.proguard.annotations.DoNotStrip
 import com.facebook.react.bridge.ReactApplicationContext
 import com.facebook.react.bridge.ReactMethod
 import com.facebook.react.common.annotations.FrameworkAPI
 import com.facebook.react.module.annotations.ReactModule
 import com.facebook.react.turbomodule.core.CallInvokerHolderImpl
+import java.io.InputStream
+import java.net.URL
 
 @OptIn(FrameworkAPI::class)
 @ReactModule(name = ETInstaller.NAME)
@@ -14,6 +17,21 @@ class ETInstaller(
 ) : NativeETInstallerSpec(reactContext) {
   companion object {
     const val NAME = NativeETInstallerSpec.NAME
+
+    @JvmStatic
+    @DoNotStrip
+    @Throws(Exception::class)
+    fun fetchByteDataFromUrl(source: String): ByteArray {
+      val url = URL(source)
+      val connection = url.openConnection()
+      connection.connect()
+
+      val inputStream: InputStream = connection.getInputStream()
+      val data = inputStream.readBytes()
+      inputStream.close()
+
+      return data
+    }
   }
 
   private val mHybridData: HybridData
@@ -32,7 +50,7 @@ class ETInstaller(
       val jsCallInvokerHolder = reactContext.jsCallInvokerHolder as CallInvokerHolderImpl
       mHybridData = initHybrid(reactContext.javaScriptContextHolder!!.get(), jsCallInvokerHolder)
     } catch (exception: UnsatisfiedLinkError) {
-      throw RuntimeException("Could not load native module Install", exception)
+      throw RuntimeException("Could not load native module ETInstaller", exception)
     }
   }
 

android/src/main/java/com/swmansion/rnexecutorch/ETInstaller.kt

@@ -18,8 +18,6 @@ class RnExecutorchPackage : TurboReactPackage() {
       LLM(reactContext)
     } else if (name == ETModule.NAME) {
       ETModule(reactContext)
-    } else if (name == StyleTransfer.NAME) {
-      StyleTransfer(reactContext)
     } else if (name == Classification.NAME) {
       Classification(reactContext)
     } else if (name == ObjectDetection.NAME) {
@@ -66,17 +64,6 @@ class RnExecutorchPackage : TurboReactPackage() {
           true,
         )
 
-      moduleInfos[StyleTransfer.NAME] =
-        ReactModuleInfo(
-          StyleTransfer.NAME,
-          StyleTransfer.NAME,
-          false, // canOverrideExistingModule
-          false, // needsEagerInit
-          true, // hasConstants
-          false, // isCxxModule
-          true,
-        )
-
       moduleInfos[Classification.NAME] =
         ReactModuleInfo(
           Classification.NAME,