feat: port style transfer implementation to C++ (#229)

Port native implementation of style transfer to C++ #227. This is a part
of a larger effort to merge native code from Kotlin and Objective C to a
single implementation in C++.

Old style transfer modules have been replaced by new, stateful
implementation. For this purpose non-static versions of useModule and
BaseModule have been introduced.
Old implementations of style transfer have been removed. Added methods
for handling fetching data via https that are passed on as function
objects to C++.
- Ported image processing capabilities equivalent to the ObjC/Kotlin
implementations.
- `ModelHostObject` serves as an automatic interface between model
implementations and JSI. This is done by template metaprogramming;
defining methods in `JsiConversions.h` for types used by the model is
necessary for it to work.
- A factory method for loading a style transfer model is registered in
`RnExecutorchInstaller`.

- `common/rnexecutorch/jsi` originates from react-native-audio-api
- `base64.*` is used for converting from base64 due to
https://renenyffenegger.ch/notes/development/Base64/Encoding-and-decoding-base-64-with-cpp/
- `ada` is a header-only library for parsing urls due to
https://github.com/ada-url/ada
- headers for Executorch 0.6 have been updated
- OpenCV 4.11.0 dependency is introduced for Android/C++. For iOS the
version is bumped via Cocoapods to 4.11.0.
- C10 headers are the dependency of OpenCV.

- Exceptions in native C++ do not result in promise getting rejected
when forwarding yet.
- Garbage collection can malfunction for host objects. C++ host objects
need to notify JS runtime about their size via external memory pressure
for the garbage collector to know to free them. This is not yet done.

Author: JakubGonera

common/ada/ada.cpp

@@ -1,7 +1,7 @@
-#pragma once
+#include "Log.h"
 
 #include <cstdarg>
-#include <string>
+#include <cstdio>
 
 #ifdef __ANDROID__
 #include <android/log.h>
@@ -12,23 +12,20 @@
 
 namespace rnexecutorch {
 
-enum class LOG_LEVEL { INFO, ERROR, DEBUG };
-
 #ifdef __ANDROID__
 android_LogPriority androidLogLevel(LOG_LEVEL logLevel) {
   switch (logLevel) {
-  case LOG_LEVEL::INFO:
+  case LOG_LEVEL::Info:
   default:
     return ANDROID_LOG_INFO;
-  case LOG_LEVEL::ERROR:
+  case LOG_LEVEL::Error:
     return ANDROID_LOG_ERROR;
-  case LOG_LEVEL::DEBUG:
+  case LOG_LEVEL::Debug:
     return ANDROID_LOG_DEBUG;
   }
 }
 #endif
 
-// const char* instead of const std::string& as va_start doesn't take references
 void log(LOG_LEVEL logLevel, const char *fmt, ...) {
   va_list args;
   va_start(args, fmt);
@@ -52,14 +49,14 @@ void log(LOG_LEVEL logLevel, const char *fmt, ...) {
 #ifdef __APPLE__
 
   switch (logLevel) {
-  case LOG_LEVEL::INFO:
+  case LOG_LEVEL::Info:
   default:
     os_log_info(OS_LOG_DEFAULT, "%s", buf);
     break;
-  case LOG_LEVEL::ERROR:
+  case LOG_LEVEL::Error:
     os_log_error(OS_LOG_DEFAULT, "%s", buf);
     break;
-  case LOG_LEVEL::DEBUG:
+  case LOG_LEVEL::Debug:
     os_log_debug(OS_LOG_DEFAULT, "%s", buf);
     break;
   }
@@ -68,4 +65,4 @@ void log(LOG_LEVEL logLevel, const char *fmt, ...) {
   va_end(args);
 }
 
-} // namespace rnexecutorch
+} // namespace rnexecutorch

common/ada/ada.h

@@ -0,0 +1,10 @@
+#pragma once
+
+namespace rnexecutorch {
+
+enum class LOG_LEVEL { Info, Error, Debug };
+
+// const char* instead of const std::string& as va_start doesn't take references
+void log(LOG_LEVEL logLevel, const char *fmt, ...);
+
+} // namespace rnexecutorch

common/Log.h common/rnexecutorch/Log.cppcommon/Log.h renamed to common/rnexecutorch/Log.cpp

@@ -0,0 +1,61 @@
+#include "RnExecutorchInstaller.h"
+
+#include <rnexecutorch/host_objects/JsiConversions.h>
+#include <rnexecutorch/host_objects/ModelHostObject.h>
+#include <rnexecutorch/jsi/JsiPromise.h>
+#include <rnexecutorch/models/StyleTransfer.h>
+
+namespace rnexecutorch {
+
+// This function fetches data from a url address. It is implemented in
+// Kotlin/ObjectiveC++ and then bound to this variable. It's done to not handle
+// SSL intricacies manually, as it is done automagically in ObjC++/Kotlin.
+FetchUrlFunc_t fetchUrlFunc;
+
+jsi::Function RnExecutorchInstaller::loadStyleTransfer(
+    jsi::Runtime *jsiRuntime,
+    const std::shared_ptr<react::CallInvoker> &jsCallInvoker) {
+  return jsi::Function::createFromHostFunction(
+      *jsiRuntime, jsi::PropNameID::forAscii(*jsiRuntime, "loadStyleTransfer"),
+      0,
+      [jsCallInvoker](jsi::Runtime &runtime, const jsi::Value &thisValue,
+                      const jsi::Value *args, size_t count) -> jsi::Value {
+        assert(count == 1);
+        try {
+          auto source = jsiconversion::getValue<std::string>(args[0], runtime);
+
+          auto styleTransferPtr =
+              std::make_shared<StyleTransfer>(source, &runtime);
+          auto styleTransferHostObject =
+              std::make_shared<ModelHostObject<StyleTransfer>>(
+                  styleTransferPtr, &runtime, jsCallInvoker);
+
+          return jsi::Object::createFromHostObject(runtime,
+                                                   styleTransferHostObject);
+        } catch (const std::runtime_error &e) {
+          // This catch should be merged with the next one
+          // (std::runtime_error inherits from std::exception) HOWEVER react
+          // native has broken RTTI which breaks proper exception type
+          // checking. Remove when the following change is present in our
+          // version:
+          // https://github.com/facebook/react-native/commit/3132cc88dd46f95898a756456bebeeb6c248f20e
+          throw jsi::JSError(runtime, e.what());
+        } catch (const std::exception &e) {
+          throw jsi::JSError(runtime, e.what());
+        } catch (...) {
+          throw jsi::JSError(runtime, "Unknown error");
+        }
+      });
+}
+
+void RnExecutorchInstaller::injectJSIBindings(
+    jsi::Runtime *jsiRuntime,
+    const std::shared_ptr<react::CallInvoker> &jsCallInvoker,
+    FetchUrlFunc_t fetchDataFromUrl) {
+  fetchUrlFunc = fetchDataFromUrl;
+
+  jsiRuntime->global().setProperty(
+      *jsiRuntime, "loadStyleTransfer",
+      loadStyleTransfer(jsiRuntime, jsCallInvoker));
+}
+} // namespace rnexecutorch

common/rnexecutorch/Log.h

@@ -4,23 +4,26 @@
 #include <string>
 #include <thread>
 
+#include <ReactCommon/CallInvoker.h>
 #include <jsi/jsi.h>
 
-#include "jsi/JsiPromise.h"
-
 namespace rnexecutorch {
 
+using FetchUrlFunc_t = std::function<std::vector<std::byte>(std::string)>;
+
 using namespace facebook;
 
 class RnExecutorchInstaller {
 public:
   static void
   injectJSIBindings(jsi::Runtime *jsiRuntime,
-                    const std::shared_ptr<react::CallInvoker> &jsCallInvoker) {
-    // Install JSI methods here
-  }
+                    const std::shared_ptr<react::CallInvoker> &jsCallInvoker,
+                    FetchUrlFunc_t fetchDataFromUrl);
 
 private:
+  static jsi::Function
+  loadStyleTransfer(jsi::Runtime *jsiRuntime,
+                    const std::shared_ptr<react::CallInvoker> &jsCallInvoker);
 };
 
 } // namespace rnexecutorch

common/rnexecutorch/RnExecutorchInstaller.cpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <chrono>
+#include <string>
+
+namespace rnexecutorch::fileutils {
+
+inline std::string getTimeID() {
+  return std::to_string(std::chrono::duration_cast<std::chrono::milliseconds>(
+                            std::chrono::system_clock::now().time_since_epoch())
+                            .count());
+}
+
+} // namespace rnexecutorch::fileutils

common/RnExecutorchInstaller.h …mon/rnexecutorch/RnExecutorchInstaller.hcommon/RnExecutorchInstaller.h renamed to common/rnexecutorch/RnExecutorchInstaller.h common/RnExecutorchInstaller.h renamed to common/rnexecutorch/RnExecutorchInstaller.h

@@ -0,0 +1,127 @@
+#include "ImageProcessing.h"
+
+#include <chrono>
+#include <filesystem>
+
+#include <ada/ada.h>
+
+#include <rnexecutorch/Log.h>
+#include <rnexecutorch/RnExecutorchInstaller.h>
+#include <rnexecutorch/data_processing/FileUtils.h>
+#include <rnexecutorch/data_processing/base64.h>
+
+namespace rnexecutorch {
+// This is defined in RnExecutorchInstaller.cpp. This function fetches data
+// from a url address. It is implemented in Kotlin/ObjectiveC++ and then bound
+// to this variable. It's done to not handle SSL intricacies manually, as it is
+// done automagically in ObjC++/Kotlin.
+extern FetchUrlFunc_t fetchUrlFunc;
+namespace imageprocessing {
+std::vector<float> colorMatToVector(const cv::Mat &mat) {
+  return colorMatToVector(mat, cv::Scalar(0.0, 0.0, 0.0),
+                          cv::Scalar(1.0, 1.0, 1.0));
+}
+
+std::vector<float> colorMatToVector(const cv::Mat &mat, cv::Scalar mean,
+                                    cv::Scalar variance) {
+  int pixelCount = mat.cols * mat.rows;
+  std::vector<float> v(pixelCount * 3);
+
+  for (int i = 0; i < pixelCount; i++) {
+    int row = i / mat.cols;
+    int col = i % mat.cols;
+    cv::Vec3b pixel = mat.at<cv::Vec3b>(row, col);
+    v[0 * pixelCount + i] =
+        (pixel[0] - mean[0] * 255.0) / (variance[0] * 255.0);
+    v[1 * pixelCount + i] =
+        (pixel[1] - mean[1] * 255.0) / (variance[1] * 255.0);
+    v[2 * pixelCount + i] =
+        (pixel[2] - mean[2] * 255.0) / (variance[2] * 255.0);
+  }
+
+  return v;
+}
+
+cv::Mat bufferToColorMat(const std::span<const float> &buffer,
+                         cv::Size matSize) {
+  cv::Mat mat(matSize, CV_8UC3);
+
+  int pixelCount = matSize.width * matSize.height;
+  for (int i = 0; i < pixelCount; i++) {
+    int row = i / matSize.width;
+    int col = i % matSize.width;
+
+    float r = buffer[0 * pixelCount + i];
+    float g = buffer[1 * pixelCount + i];
+    float b = buffer[2 * pixelCount + i];
+
+    cv::Vec3b color(static_cast<uchar>(b * 255), static_cast<uchar>(g * 255),
+                    static_cast<uchar>(r * 255));
+    mat.at<cv::Vec3b>(row, col) = color;
+  }
+
+  return mat;
+}
+
+std::string saveToTempFile(const cv::Mat &image) {
+  std::string filename = "rn_executorch_" + fileutils::getTimeID() + ".png";
+
+  std::filesystem::path tempDir = std::filesystem::temp_directory_path();
+  std::filesystem::path filePath = tempDir / filename;
+
+  if (!cv::imwrite(filePath.string(), image)) {
+    throw std::runtime_error("Failed to save the image: " + filePath.string());
+  }
+
+  return "file://" + filePath.string();
+}
+
+cv::Mat readImage(const std::string &imageURI) {
+  cv::Mat image;
+
+  if (imageURI.starts_with("data")) {
+    // base64
+    std::stringstream uriStream(imageURI);
+    std::string stringData;
+    std::size_t segmentIndex{0};
+    while (std::getline(uriStream, stringData, ',')) {
+      ++segmentIndex;
+    }
+    if (segmentIndex != 1) {
+      throw std::runtime_error("Read image error: invalid base64 URI");
+    }
+    auto data = base64_decode(stringData);
+    cv::Mat encodedData(1, data.size(), CV_8UC1, (void *)data.data());
+    image = cv::imdecode(encodedData, cv::IMREAD_COLOR);
+  } else if (imageURI.starts_with("file")) {
+    // local file
+    auto url = ada::parse(imageURI);
+    image = cv::imread(std::string{url->get_pathname()}, cv::IMREAD_COLOR);
+  } else {
+    // remote file
+    std::vector<std::byte> imageData = fetchUrlFunc(imageURI);
+    image = cv::imdecode(
+        cv::Mat(1, imageData.size(), CV_8UC1, (void *)imageData.data()),
+        cv::IMREAD_COLOR);
+  }
+
+  if (image.empty()) {
+    throw std::runtime_error("Read image error: invalid argument");
+  }
+
+  return image;
+}
+
+TensorPtr getTensorFromMatrix(const std::vector<int32_t> &sizes,
+                              const cv::Mat &matrix) {
+  std::vector<float> inputVector = colorMatToVector(matrix);
+  return executorch::extension::make_tensor_ptr(sizes, inputVector);
+}
+
+cv::Mat getMatrixFromTensor(cv::Size size, const Tensor &tensor) {
+  auto resultData = static_cast<const float *>(tensor.const_data_ptr());
+  return bufferToColorMat(std::span<const float>(resultData, tensor.numel()),
+                          size);
+}
+} // namespace imageprocessing
+} // namespace rnexecutorch

common/rnexecutorch/data_processing/FileUtils.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <executorch/extension/tensor/tensor.h>
+#include <executorch/extension/tensor/tensor_ptr.h>
+#include <opencv2/opencv.hpp>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace rnexecutorch::imageprocessing {
+using executorch::aten::Tensor;
+using executorch::extension::TensorPtr;
+
+/// @brief Convert a OpenCV matrix to channel-first vector representation
+std::vector<float> colorMatToVector(const cv::Mat &mat, cv::Scalar mean,
+                                    cv::Scalar variance);
+/// @brief Convert a OpenCV matrix to channel-first vector representation
+std::vector<float> colorMatToVector(const cv::Mat &mat);
+/// @brief Convert a channel-first representation of an RGB image to OpenCV
+/// matrix
+cv::Mat bufferToColorMat(const std::span<const float> &buffer,
+                         cv::Size matSize);
+std::string saveToTempFile(const cv::Mat &image);
+cv::Mat readImage(const std::string &imageURI);
+TensorPtr getTensorFromMatrix(const std::vector<int32_t> &sizes,
+                              const cv::Mat &mat);
+cv::Mat getMatrixFromTensor(cv::Size size, const Tensor &tensor);
+
+} // namespace rnexecutorch::imageprocessing