feat: add support for hugging face tokenizers, add executorch and tokenizers as submodules

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "executorch"]
+    path = third-party/executorch
+    url = https://github.com/software-mansion-labs/executorch
+[submodule "tokenizers-cpp"]
+	path = third-party/tokenizers-cpp
+	url = https://github.com/software-mansion-labs/tokenizers-cpp

android/build.gradle

@@ -103,7 +103,7 @@ dependencies {
   implementation "com.facebook.react:react-android:+"
   implementation 'org.opencv:opencv:4.10.0'
   implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
-  implementation 'com.github.software-mansion:react-native-executorch:main-SNAPSHOT'
+  implementation(files("libs/executorch.aar"))
   implementation 'org.opencv:opencv:4.10.0'
   implementation("com.squareup.okhttp3:okhttp:4.9.2")
 }

android/src/main/java/com/swmansion/rnexecutorch/RnExecutorchPackage.kt

@@ -32,6 +32,8 @@ class RnExecutorchPackage : TurboReactPackage() {
       VerticalOCR(reactContext)
     } else if (name == ImageSegmentation.NAME) {
       ImageSegmentation(reactContext)
+    } else if (name == Tokenizer.NAME) {
+      Tokenizer(reactContext)
     } else {
       null
     }
@@ -118,12 +120,25 @@ class RnExecutorchPackage : TurboReactPackage() {
           true,
         )
 
-      moduleInfos[ImageSegmentation.NAME] = ReactModuleInfo(
-        ImageSegmentation.NAME, ImageSegmentation.NAME, false,  // canOverrideExistingModule
-        false,  // needsEagerInit
-        false,  // isCxxModule
-        true
-      )
+      moduleInfos[ImageSegmentation.NAME] =
+        ReactModuleInfo(
+          ImageSegmentation.NAME,
+          ImageSegmentation.NAME,
+          false, // canOverrideExistingModule
+          false, // needsEagerInit
+          false, // isCxxModule
+          true,
+        )
+
+      moduleInfos[Tokenizer.NAME] =
+        ReactModuleInfo(
+          Tokenizer.NAME,
+          Tokenizer.NAME,
+          false, // canOverrideExistingModule
+          false, // needsEagerInit
+          false, // isCxxModule
+          true,
+        )
       moduleInfos
     }
 }

android/src/main/java/com/swmansion/rnexecutorch/Tokenizer.kt

@@ -0,0 +1,66 @@
+package com.swmansion.rnexecutorch
+
+import com.facebook.react.bridge.Promise
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.bridge.ReadableArray
+import com.swmansion.rnexecutorch.utils.ArrayUtils.Companion.createIntArray
+import com.swmansion.rnexecutorch.utils.ArrayUtils.Companion.createReadableArrayFromIntArray
+import com.swmansion.rnexecutorch.utils.ETError
+import org.pytorch.executorch.HuggingFaceTokenizer
+import java.net.URL
+
+class Tokenizer(
+  reactContext: ReactApplicationContext,
+) : NativeTokenizerSpec(reactContext) {
+  private lateinit var tokenizer: HuggingFaceTokenizer
+
+  companion object {
+    const val NAME = "Tokenizer"
+  }
+
+  override fun load(
+    tokenizerSource: String,
+    promise: Promise,
+  ) {
+    try {
+      tokenizer = HuggingFaceTokenizer(URL(tokenizerSource).path)
+      promise.resolve(0)
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
+    }
+  }
+
+  override fun decode(
+    input: ReadableArray,
+    promise: Promise,
+  ) {
+    promise.resolve(tokenizer.decode(createIntArray(input)))
+  }
+
+  override fun encode(
+    input: String,
+    promise: Promise,
+  ) {
+    promise.resolve(createReadableArrayFromIntArray(tokenizer.encode(input)))
+  }
+
+  override fun getVocabSize(promise: Promise) {
+    promise.resolve(tokenizer.vocabSize)
+  }
+
+  override fun idToToken(
+    id: Double,
+    promise: Promise,
+  ) {
+    promise.resolve(tokenizer.idToToken(id.toInt()))
+  }
+
+  override fun tokenToId(
+    token: String,
+    promise: Promise,
+  ) {
+    promise.resolve(tokenizer.tokenToId(token))
+  }
+
+  override fun getName(): String = NAME
+}

ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/Headers/HuggingFaceTokenizer.h

@@ -0,0 +1,12 @@
+#import <Foundation/Foundation.h>
+
+@interface HuggingFaceTokenizer : NSObject
+
+- (instancetype)initWithTokenizerPath:(NSString *)tokenizerPath;
+- (NSArray<NSNumber *> *)encode:(NSString *)text;
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds;
+- (NSUInteger)getVocabSize;
+- (NSString *)idToToken:(NSInteger)tokenId;
+- (NSInteger)tokenToId:(NSString *)token;
+
+@end

ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/_CodeSignature/CodeResources

@@ -6,15 +6,19 @@
 	<dict>
 		<key>Headers/ETModel.h</key>
 		<data>
-		CFAz750OjepOG7MVBPABGfKHNeI=
+		uR77dUqxDWLwIE0h9dvXHSjMFWo=
+		</data>
+		<key>Headers/HuggingFaceTokenizer.h</key>
+		<data>
+		cVZsliuTmV2umgK601d6PamTRSw=
 		</data>
 		<key>Headers/LLaMARunner.h</key>
 		<data>
-		SU8Fo2gR+gVVl9IplHgBJBRh1gQ=
+		7OcOQPzHkK7faqsbtXBztEr9VO0=
 		</data>
 		<key>Info.plist</key>
 		<data>
-		l3rE2nBARVh++WIyHCfeHXD6Ewo=
+		wBJJ2TZXoQXEBBiUjBovczXIsbU=
 		</data>
 	</dict>
 	<key>files2</key>
@@ -23,14 +27,21 @@
 		<dict>
 			<key>hash2</key>
 			<data>
-			UXFd6a5OARqV4JnB+Jm4uqmt15aUmnXSOLPQKZTWZCc=
+			+Ty+KzH7+xNA5B9kfmD44hgTJKIZuk2qN1kauF6thOw=
+			</data>
+		</dict>
+		<key>Headers/HuggingFaceTokenizer.h</key>
+		<dict>
+			<key>hash2</key>
+			<data>
+			0ETM5qw12+W8ULx2zP2UkVomFrBRnwAr8I7po2ACk/k=
 			</data>
 		</dict>
 		<key>Headers/LLaMARunner.h</key>
 		<dict>
 			<key>hash2</key>
 			<data>
-			or8gFkCO2QVkQgaeFAaqs/WqGjv8kABL8Rzcdcuexw0=
+			AZECdFqfkT4YWu8Nqga5ALaJUme8IOAnhOaAUM+iOvU=
 			</data>
 		</dict>
 	</dict>

ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/Headers/HuggingFaceTokenizer.h

@@ -0,0 +1,12 @@
+#import <Foundation/Foundation.h>
+
+@interface HuggingFaceTokenizer : NSObject
+
+- (instancetype)initWithTokenizerPath:(NSString *)tokenizerPath;
+- (NSArray<NSNumber *> *)encode:(NSString *)text;
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds;
+- (NSUInteger)getVocabSize;
+- (NSString *)idToToken:(NSInteger)tokenId;
+- (NSInteger)tokenToId:(NSString *)token;
+
+@end

ios/ExecutorchLib.xcframework/ios-arm64/ExecutorchLib.framework/_CodeSignature/CodeResources

@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>files</key>
+	<dict>
+		<key>Headers/ETModel.h</key>
+		<data>
+		uR77dUqxDWLwIE0h9dvXHSjMFWo=
+		</data>
+		<key>Headers/HuggingFaceTokenizer.h</key>
+		<data>
+		cVZsliuTmV2umgK601d6PamTRSw=
+		</data>
+		<key>Headers/LLaMARunner.h</key>
+		<data>
+		7OcOQPzHkK7faqsbtXBztEr9VO0=
+		</data>
+		<key>Info.plist</key>
+		<data>
+		xUwVRDNEWZOSxtl3vg4xATbAoi8=
+		</data>
+	</dict>
+	<key>files2</key>
+	<dict>
+		<key>Headers/ETModel.h</key>
+		<dict>
+			<key>hash2</key>
+			<data>
+			+Ty+KzH7+xNA5B9kfmD44hgTJKIZuk2qN1kauF6thOw=
+			</data>
+		</dict>
+		<key>Headers/HuggingFaceTokenizer.h</key>
+		<dict>
+			<key>hash2</key>
+			<data>
+			0ETM5qw12+W8ULx2zP2UkVomFrBRnwAr8I7po2ACk/k=
+			</data>
+		</dict>
+		<key>Headers/LLaMARunner.h</key>
+		<dict>
+			<key>hash2</key>
+			<data>
+			AZECdFqfkT4YWu8Nqga5ALaJUme8IOAnhOaAUM+iOvU=
+			</data>
+		</dict>
+	</dict>
+	<key>rules</key>
+	<dict>
+		<key>^.*</key>
+		<true/>
+		<key>^.*\.lproj/</key>
+		<dict>
+			<key>optional</key>
+			<true/>
+			<key>weight</key>
+			<real>1000</real>
+		</dict>
+		<key>^.*\.lproj/locversion.plist$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>1100</real>
+		</dict>
+		<key>^Base\.lproj/</key>
+		<dict>
+			<key>weight</key>
+			<real>1010</real>
+		</dict>
+		<key>^version.plist$</key>
+		<true/>
+	</dict>
+	<key>rules2</key>
+	<dict>
+		<key>.*\.dSYM($|/)</key>
+		<dict>
+			<key>weight</key>
+			<real>11</real>
+		</dict>
+		<key>^(.*/)?\.DS_Store$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>2000</real>
+		</dict>
+		<key>^.*</key>
+		<true/>
+		<key>^.*\.lproj/</key>
+		<dict>
+			<key>optional</key>
+			<true/>
+			<key>weight</key>
+			<real>1000</real>
+		</dict>
+		<key>^.*\.lproj/locversion.plist$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>1100</real>
+		</dict>
+		<key>^Base\.lproj/</key>
+		<dict>
+			<key>weight</key>
+			<real>1010</real>
+		</dict>
+		<key>^Info\.plist$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+		<key>^PkgInfo$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+		<key>^embedded\.provisionprofile$</key>
+		<dict>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+		<key>^version\.plist$</key>
+		<dict>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+	</dict>
+</dict>
+</plist>

ios/RnExecutorch/Tokenizer.h

@@ -0,0 +1,5 @@
+#import <RnExecutorchSpec/RnExecutorchSpec.h>
+
+@interface Tokenizer : NSObject <NativeTokenizerSpec>
+
+@end

ios/RnExecutorch/Tokenizer.mm

@@ -0,0 +1,58 @@
+#import "Tokenizer.h"
+#import <ExecutorchLib/HuggingFaceTokenizer.h>
+#import <React/RCTBridgeModule.h>
+
+@implementation Tokenizer {
+  HuggingFaceTokenizer *tokenizer;
+}
+
+RCT_EXPORT_MODULE()
+
+- (void)load:(NSString *)tokenizerSource
+     resolve:(RCTPromiseResolveBlock)resolve
+      reject:(RCTPromiseRejectBlock)reject {
+  @try {
+    tokenizer = [[HuggingFaceTokenizer alloc]
+        initWithTokenizerPath:[NSURL URLWithString:tokenizerSource].path];
+    resolve(@(0));
+  } @catch (NSException *exception) {
+    reject(@"Tokenizer_Error", @"Failed to load tokenizer", nil);
+  }
+}
+
+- (void)encode:(NSString *)input
+       resolve:(RCTPromiseResolveBlock)resolve
+        reject:(RCTPromiseRejectBlock)reject {
+  resolve([tokenizer encode:input]);
+}
+
+- (void)decode:(NSArray *)input
+       resolve:(RCTPromiseResolveBlock)resolve
+        reject:(RCTPromiseRejectBlock)reject {
+  resolve([tokenizer decode:input]);
+}
+
+- (void)getVocabSize:(RCTPromiseResolveBlock)resolve
+              reject:(RCTPromiseRejectBlock)reject {
+  resolve([NSNumber numberWithUnsignedInteger:[tokenizer getVocabSize]]);
+}
+
+- (void)idToToken:(double)input
+          resolve:(RCTPromiseResolveBlock)resolve
+           reject:(RCTPromiseRejectBlock)reject {
+  NSInteger tokenID = (NSInteger)input;
+  resolve([tokenizer idToToken:tokenID]);
+}
+
+- (void)tokenToId:(NSString *)input
+          resolve:(RCTPromiseResolveBlock)resolve
+           reject:(RCTPromiseRejectBlock)reject {
+  resolve([NSNumber numberWithInteger:[tokenizer tokenToId:input]]);
+}
+
+- (std::shared_ptr<facebook::react::TurboModule>)getTurboModule:
+    (const facebook::react::ObjCTurboModule::InitParams &)params {
+  return std::make_shared<facebook::react::NativeTokenizerSpecJSI>(params);
+}
+
+@end