add not working jsi bindings

Author: mlodyjesienin

packages/react-native-executorch/common/rnexecutorch/RnExecutorchInstaller.h

@@ -36,7 +36,8 @@ REGISTER_CONSTRUCTOR(TextEmbeddings, std::string, std::string,
                      std::shared_ptr<react::CallInvoker>);
 REGISTER_CONSTRUCTOR(LLM, std::string, std::string,
                      std::shared_ptr<react::CallInvoker>);
-
+REGISTER_CONSTRUCTOR(OCR, std::string, std::string, std::string, std::string,
+                     std::string, std::shared_ptr<react::CallInvoker>);
 using namespace facebook;
 
 class RnExecutorchInstaller {
@@ -55,7 +56,6 @@ class RnExecutorchInstaller {
   loadModel(jsi::Runtime *jsiRuntime,
             std::shared_ptr<react::CallInvoker> jsCallInvoker,
             const std::string &loadFunctionName) {
-
     return jsi::Function::createFromHostFunction(
         *jsiRuntime, jsi::PropNameID::forAscii(*jsiRuntime, loadFunctionName),
         0,

packages/react-native-executorch/common/rnexecutorch/host_objects/JsiConversions.h

@@ -13,6 +13,7 @@
 #include <rnexecutorch/metaprogramming/TypeConcepts.h>
 #include <rnexecutorch/models/object_detection/Constants.h>
 #include <rnexecutorch/models/object_detection/Utils.h>
+#include <rnexecutorch/models/ocr/Types.h>
 
 namespace rnexecutorch::jsiconversion {
 
@@ -266,4 +267,25 @@ inline jsi::Value getJsiValue(const std::vector<Detection> &detections,
   return array;
 }
 
+inline jsi::Value getJsiValue(const std::vector<float> &vec,
+                              jsi::Runtime &runtime) {
+  jsi::Array array(runtime, vec.size());
+  for (size_t i = 0; i < vec.size(); i++) {
+    array.setValueAtIndex(runtime, i, jsi::Value(static_cast<float>(vec[i])));
+  }
+  return jsi::Value(runtime, array);
+}
+
+inline jsi::Value getJsiValue(const std::vector<OCRDetection> &detections,
+                              jsi::Runtime &runtime) {
+  jsi::Array array(runtime, detections.size());
+  for (std::size_t i = 0; i < detections.size(); ++i) {
+    jsi::Object detection(runtime);
+    jsi::Object bbox(runtime);
+    detection.setProperty(runtime, "text", detections[i].text);
+    detection.setProperty(runtime, "score", detections[i].score);
+    array.setValueAtIndex(runtime, i, detection);
+  }
+  return array;
+}
 } // namespace rnexecutorch::jsiconversion

packages/react-native-executorch/src/controllers/OCRController.ts

@@ -1,12 +1,11 @@
 import { symbols } from '../constants/ocr/symbols';
 import { ETError, getError } from '../Error';
-import { OCRNativeModule } from '../native/RnExecutorchModules';
 import { ResourceSource } from '../types/common';
 import { OCRLanguage } from '../types/ocr';
 import { ResourceFetcher } from '../utils/ResourceFetcher';
 
 export class OCRController {
-  private nativeModule: typeof OCRNativeModule;
+  private nativeModule: any; //typeof OCRNativeModule;
   public isReady: boolean = false;
   public isGenerating: boolean = false;
   public error: string | null = null;
@@ -21,14 +20,14 @@ export class OCRController {
     isGeneratingCallback = (_isGenerating: boolean) => {},
     errorCallback = (_error: string) => {},
   }) {
-    this.nativeModule = OCRNativeModule;
+    //this.nativeModule = OCRNativeModule;
     this.modelDownloadProgressCallback = modelDownloadProgressCallback;
     this.isReadyCallback = isReadyCallback;
     this.isGeneratingCallback = isGeneratingCallback;
     this.errorCallback = errorCallback;
   }
 
-  public loadModel = async (
+  public load = async (
     detectorSource: ResourceSource,
     recognizerSources: {
       recognizerLarge: ResourceSource;
@@ -58,14 +57,23 @@ export class OCRController {
       if (paths === null || paths?.length < 4) {
         throw new Error('Download interrupted!');
       }
-      await this.nativeModule.loadModule(
+      console.log('fetching ocr finished!', paths[3]);
+      //load OCR from cpp model
+      this.nativeModule = global.loadOCR(
         paths[0]!,
         paths[1]!,
         paths[2]!,
         paths[3]!,
         symbols[language]
       );
-
+      // await this.nativeModule.loadModule(
+      //   paths[0]!,
+      //   paths[1]!,
+      //   paths[2]!,
+      //   paths[3]!,
+      //   symbols[language]
+      // );
+      console.log('loaded ocr finished!,');
       this.isReady = true;
       this.isReadyCallback(this.isReady);
     } catch (e) {
@@ -88,7 +96,7 @@ export class OCRController {
     try {
       this.isGenerating = true;
       this.isGeneratingCallback(this.isGenerating);
-      return await this.nativeModule.forward(input);
+      return await this.nativeModule.generate(input);
     } catch (e) {
       throw new Error(getError(e));
     } finally {

packages/react-native-executorch/src/hooks/computer_vision/useOCR.ts

@@ -44,7 +44,7 @@ export const useOCR = ({
 
   useEffect(() => {
     const loadModel = async () => {
-      await model.loadModel(detectorSource, recognizerSources, language);
+      await model.load(detectorSource, recognizerSources, language);
     };
 
     if (!preventLoad) {

packages/react-native-executorch/src/hooks/useNonStaticModule.ts

@@ -29,6 +29,7 @@ export const useNonStaticModule = <
   const [isReady, setIsReady] = useState(false);
   const [isGenerating, setIsGenerating] = useState(false);
   const [downloadProgress, setDownloadProgress] = useState(0);
+  console.log(module);
   const model = useMemo(() => new module(), [module]);
 
   useEffect(() => {

packages/react-native-executorch/src/index.tsx

@@ -13,6 +13,13 @@ declare global {
   var loadImageEmbeddings: (source: string) => any;
   var loadTextEmbeddings: (modelSource: string, tokenizerSource: string) => any;
   var loadLLM: (modelSource: string, tokenizerSource: string) => any;
+  var loadOCR: (
+    detectorSource: string,
+    recognizerLarge: string,
+    recognizerMedium: string,
+    recognizerSmall: string,
+    symbols: string
+  ) => any;
 }
 // eslint-disable no-var
 if (
@@ -24,7 +31,8 @@ if (
   global.loadTokenizerModule == null ||
   global.loadTextEmbeddings == null ||
   global.loadImageEmbeddings == null ||
-  global.loadLLM == null
+  global.loadLLM == null ||
+  global.loadOCR == null
 ) {
   if (!ETInstallerNativeModule) {
     throw new Error(

packages/react-native-executorch/src/modules/computer_vision/OCRModule.ts

@@ -20,7 +20,7 @@ export class OCRModule {
       modelDownloadProgressCallback: this.onDownloadProgressCallback,
     });
 
-    await this.module.loadModel(detectorSource, recognizerSources, language);
+    await this.module.load(detectorSource, recognizerSources, language);
   }
 
   static async forward(input: string) {