wip

Author: chmjkb

packages/react-native-executorch/common/rnexecutorch/TokenizerModule.cpp

@@ -0,0 +1,32 @@
+#include "TokenizerModule.h"
+#include <executorch/extension/module/module.h>
+#include <fstream>
+#include <rnexecutorch/Log.h>
+#include <rnexecutorch/data_processing/FileUtils.h>
+
+namespace rnexecutorch {
+using namespace facebook;
+
+TokenizerModule::TokenizerModule(
+    std::string source, std::shared_ptr<react::CallInvoker> callInvoker) {
+  auto blob = fileutils::loadBytesFromFile(source);
+  tokenizer = tokenizers::Tokenizer::FromBlobJSON(blob);
+}
+
+std::vector<int32_t> TokenizerModule::encode(std::string s) {
+  if (!tokenizer) {
+    throw std::runtime_error("Encode called on an uninitialized tokenizer!");
+  };
+  return tokenizer->Encode(s);
+}
+
+std::string TokenizerModule::decode(std::vector<int32_t> vec) {
+  if (!tokenizer) {
+    throw std::runtime_error("Decode called on an uninitialized tokenizer!");
+  }
+  return tokenizer->Decode(vec);
+}
+
+int TokenizerModule::getMemoryLowerBound() { return 1; }
+
+} // namespace rnexecutorch

packages/react-native-executorch/common/rnexecutorch/TypeConcepts.h

@@ -13,6 +13,16 @@ concept HasGenerate = requires(T t) {
   { &T::generate };
 };
 
+template <typename T>
+concept HasEncode = requires(T t) {
+  { &T::encode };
+};
+
+template <typename T>
+concept HasDecode = requires(T t) {
+  { &T::decode };
+};
+
 template <typename T>
 concept IsNumeric = std::is_arithmetic_v<T>;
 

packages/react-native-executorch/common/rnexecutorch/data_processing/FileUtils.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include <chrono>
+#include <filesystem>
+#include <fstream>
 #include <string>
 
 namespace rnexecutorch::fileutils {
@@ -11,4 +13,18 @@ inline std::string getTimeID() {
                             .count());
 }
 
+inline std::string loadBytesFromFile(const std::string &path) {
+  std::ifstream fs(path, std::ios::in | std::ios::binary);
+  if (fs.fail()) {
+    throw std::runtime_error("Failed to open tokenizer file");
+  }
+  std::string data;
+  fs.seekg(0, std::ios::end);
+  size_t size = static_cast<size_t>(fs.tellg());
+  fs.seekg(0, std::ios::beg);
+  data.resize(size);
+  fs.read(data.data(), size);
+  return data;
+};
+
 } // namespace rnexecutorch::fileutils

packages/react-native-executorch/common/rnexecutorch/host_objects/JsiConversions.h

@@ -41,6 +41,21 @@ inline std::string getValue<std::string>(const jsi::Value &val,
   return val.getString(runtime).utf8(runtime);
 }
 
+template <>
+inline std::vector<int32_t>
+getValue<std::vector<int32_t>>(const jsi::Value &val, jsi::Runtime &runtime) {
+  jsi::Array array = val.asObject(runtime).asArray(runtime);
+  size_t length = array.size(runtime);
+  std::vector<int32_t> result;
+  result.reserve(length);
+
+  for (size_t i = 0; i < length; ++i) {
+    jsi::Value element = array.getValueAtIndex(runtime, i);
+    result.push_back(getValue<int32_t>(element, runtime));
+  }
+  return result;
+}
+
 template <>
 inline JSTensorViewIn getValue<JSTensorViewIn>(const jsi::Value &val,
                                                jsi::Runtime &runtime) {

packages/react-native-executorch/common/rnexecutorch/host_objects/ModelHostObject.h

@@ -44,6 +44,18 @@ template <typename Model> class ModelHostObject : public JsiHostObject {
                                        promiseHostFunction<&Model::generate>,
                                        "generate"));
     }
+
+    if constexpr (HasEncode<Model>) {
+      addFunctions(JSI_EXPORT_FUNCTION(ModelHostObject<Model>,
+                                       promiseHostFunction<&Model::encode>,
+                                       "encode"));
+    }
+
+    if constexpr (HasDecode<Model>) {
+      addFunctions(JSI_EXPORT_FUNCTION(ModelHostObject<Model>,
+                                       promiseHostFunction<&Model::decode>,
+                                       "decode"));
+    }
   }
 
   // A generic host function that resolves a promise with a result of a

packages/react-native-executorch/src/index.tsx

@@ -9,14 +9,16 @@ declare global {
   var loadClassification: (source: string) => any;
   var loadObjectDetection: (source: string) => any;
   var loadExecutorchModule: (source: string) => any;
+  var loadTokenizerModule: (source: string) => any;
 }
 // eslint-disable no-var
 if (
   global.loadStyleTransfer == null ||
   global.loadImageSegmentation == null ||
   global.loadExecutorchModule == null ||
   global.loadClassification == null ||
-  global.loadObjectDetection == null
+  global.loadObjectDetection == null ||
+  global.loadTokenizerModule == null
 ) {
   if (!ETInstallerNativeModule) {
     throw new Error(
@@ -54,6 +56,7 @@ export * from './modules/natural_language_processing/LLMModule';
 export * from './modules/natural_language_processing/SpeechToTextModule';
 export * from './modules/natural_language_processing/TextEmbeddingsModule';
 export * from './modules/natural_language_processing/TokenizerModule';
+export * from './modules/natural_language_processing/NewTokenizerModule';
 
 // utils
 export * from './utils/ResourceFetcher';