wip

Author: chmjkb

packages/react-native-executorch/common/rnexecutorch/TokenizerModule.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <ReactCommon/CallInvoker.h>
+#include <string>
+#include <tokenizers-cpp/tokenizers_cpp.h>
+
+namespace rnexecutorch {
+using namespace facebook;
+
+class TokenizerModule {
+public:
+  std::unique_ptr<tokenizers::Tokenizer> tokenizer;
+  TokenizerModule(std::string source,
+                  std::shared_ptr<react::CallInvoker> callInvoker);
+  std::vector<int32_t> encode(std::string s);
+  std::string decode(std::vector<int32_t> vec);
+  int getMemoryLowerBound();
+};
+} // namespace rnexecutorch

packages/react-native-executorch/src/modules/natural_language_processing/NewTokenizerModule.ts

@@ -0,0 +1,26 @@
+import { ResourceSource } from '../../types/common';
+import { ResourceFetcher } from '../../utils/ResourceFetcher';
+
+export class NewTokenizerModule {
+  nativeModule: any;
+
+  async load(
+    modelSource: ResourceSource,
+    onDownloadProgressCallback: (_: number) => void = () => {}
+  ): Promise<void> {
+    const paths = await ResourceFetcher.fetchMultipleResources(
+      onDownloadProgressCallback,
+      modelSource
+    );
+    console.log(paths);
+    this.nativeModule = global.loadTokenizerModule(paths[0] || '');
+  }
+
+  async encode(s: string) {
+    return await this.nativeModule.encode(s);
+  }
+
+  async decode(tokens: number[]) {
+    return await this.nativeModule.decode(tokens);
+  }
+}