feat: add support for hugging face tokenizers, add executorch and tokenizers as submodules (#144)

## Description

This PR introduces support for hugging face tokenizers by utilizing cpp
bridge that connects native code with hf tokenizers library. It also
features configured submodules which point to our executorch and
tokenizers-cpp repositories that are stored in software mansion labs
organization.

### Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] Documentation update (improves or adds clarity to existing
documentation)

### Tested on

- [x] iOS
- [x] Android

### Testing instructions

<!-- Provide step-by-step instructions on how to test your changes.
Include setup details if necessary. -->

### Screenshots

<!-- Add screenshots here, if applicable -->

### Related issues

<!-- Link related issues here using #issue-number -->

### Checklist

- [ ] I have performed a self-review of my code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have updated the documentation accordingly
- [ ] My changes generate no new warnings

### Additional notes

<!-- Include any additional information, assumptions, or context that
reviewers might need to understand this PR. -->

Author: NorbertKlockiewicz

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "executorch"]
+    path = third-party/executorch
+    url = https://github.com/software-mansion-labs/executorch
+[submodule "tokenizers-cpp"]
+	path = third-party/tokenizers-cpp
+	url = https://github.com/software-mansion-labs/tokenizers-cpp

android/build.gradle

@@ -103,7 +103,7 @@ dependencies {
   implementation "com.facebook.react:react-android:+"
   implementation 'org.opencv:opencv:4.10.0'
   implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
-  implementation 'com.github.software-mansion:react-native-executorch:main-SNAPSHOT'
+  implementation(files("libs/executorch.aar"))
   implementation 'org.opencv:opencv:4.10.0'
   implementation("com.squareup.okhttp3:okhttp:4.9.2")
 }

android/libs/executorch.aar

@@ -32,6 +32,8 @@ class RnExecutorchPackage : TurboReactPackage() {
       VerticalOCR(reactContext)
     } else if (name == ImageSegmentation.NAME) {
       ImageSegmentation(reactContext)
+    } else if (name == Tokenizer.NAME) {
+      Tokenizer(reactContext)
     } else {
       null
     }
@@ -118,12 +120,25 @@ class RnExecutorchPackage : TurboReactPackage() {
           true,
         )
 
-      moduleInfos[ImageSegmentation.NAME] = ReactModuleInfo(
-        ImageSegmentation.NAME, ImageSegmentation.NAME, false,  // canOverrideExistingModule
-        false,  // needsEagerInit
-        false,  // isCxxModule
-        true
-      )
+      moduleInfos[ImageSegmentation.NAME] =
+        ReactModuleInfo(
+          ImageSegmentation.NAME,
+          ImageSegmentation.NAME,
+          false, // canOverrideExistingModule
+          false, // needsEagerInit
+          false, // isCxxModule
+          true,
+        )
+
+      moduleInfos[Tokenizer.NAME] =
+        ReactModuleInfo(
+          Tokenizer.NAME,
+          Tokenizer.NAME,
+          false, // canOverrideExistingModule
+          false, // needsEagerInit
+          false, // isCxxModule
+          true,
+        )
       moduleInfos
     }
 }

android/src/main/java/com/swmansion/rnexecutorch/RnExecutorchPackage.kt

@@ -0,0 +1,86 @@
+package com.swmansion.rnexecutorch
+
+import com.facebook.react.bridge.Promise
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.bridge.ReadableArray
+import com.swmansion.rnexecutorch.utils.ArrayUtils.Companion.createIntArray
+import com.swmansion.rnexecutorch.utils.ArrayUtils.Companion.createReadableArrayFromIntArray
+import com.swmansion.rnexecutorch.utils.ETError
+import org.pytorch.executorch.HuggingFaceTokenizer
+import java.net.URL
+
+class Tokenizer(
+  reactContext: ReactApplicationContext,
+) : NativeTokenizerSpec(reactContext) {
+  private lateinit var tokenizer: HuggingFaceTokenizer
+
+  companion object {
+    const val NAME = "Tokenizer"
+  }
+
+  override fun load(
+    tokenizerSource: String,
+    promise: Promise,
+  ) {
+    try {
+      tokenizer = HuggingFaceTokenizer(URL(tokenizerSource).path)
+      promise.resolve(0)
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
+    }
+  }
+
+  override fun decode(
+    input: ReadableArray,
+    promise: Promise,
+  ) {
+    try {
+      promise.resolve(tokenizer.decode(createIntArray(input)))
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.UndefinedError.toString())
+    }
+  }
+
+  override fun encode(
+    input: String,
+    promise: Promise,
+  ) {
+    try {
+      promise.resolve(createReadableArrayFromIntArray(tokenizer.encode(input)))
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.UndefinedError.toString())
+    }
+  }
+
+  override fun getVocabSize(promise: Promise) {
+    try {
+      promise.resolve(tokenizer.vocabSize)
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.UndefinedError.toString())
+    }
+  }
+
+  override fun idToToken(
+    id: Double,
+    promise: Promise,
+  ) {
+    try {
+      promise.resolve(tokenizer.idToToken(id.toInt()))
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.UndefinedError.toString())
+    }
+  }
+
+  override fun tokenToId(
+    token: String,
+    promise: Promise,
+  ) {
+    try {
+      promise.resolve(tokenizer.tokenToId(token))
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.UndefinedError.toString())
+    }
+  }
+
+  override fun getName(): String = NAME
+}

android/src/main/java/com/swmansion/rnexecutorch/Tokenizer.kt

@@ -0,0 +1,12 @@
+#import <Foundation/Foundation.h>
+
+@interface HuggingFaceTokenizer : NSObject
+
+- (instancetype)initWithTokenizerPath:(NSString *)tokenizerPath;
+- (NSArray<NSNumber *> *)encode:(NSString *)text;
+- (NSString *)decode:(NSArray<NSNumber *> *)tokenIds;
+- (NSUInteger)getVocabSize;
+- (NSString *)idToToken:(NSInteger)tokenId;
+- (NSInteger)tokenToId:(NSString *)token;
+
+@end

ios/ExecutorchLib.xcframework/ios-arm64-simulator/ExecutorchLib.framework/ExecutorchLib

@@ -6,15 +6,19 @@
 	<dict>
 		<key>Headers/ETModel.h</key>
 		<data>
-		CFAz750OjepOG7MVBPABGfKHNeI=
+		uR77dUqxDWLwIE0h9dvXHSjMFWo=
+		</data>
+		<key>Headers/HuggingFaceTokenizer.h</key>
+		<data>
+		cVZsliuTmV2umgK601d6PamTRSw=
 		</data>
 		<key>Headers/LLaMARunner.h</key>
 		<data>
-		SU8Fo2gR+gVVl9IplHgBJBRh1gQ=
+		7OcOQPzHkK7faqsbtXBztEr9VO0=
 		</data>
 		<key>Info.plist</key>
 		<data>
-		l3rE2nBARVh++WIyHCfeHXD6Ewo=
+		wBJJ2TZXoQXEBBiUjBovczXIsbU=
 		</data>
 	</dict>
 	<key>files2</key>
@@ -23,14 +27,21 @@
 		<dict>
 			<key>hash2</key>
 			<data>
-			UXFd6a5OARqV4JnB+Jm4uqmt15aUmnXSOLPQKZTWZCc=
+			+Ty+KzH7+xNA5B9kfmD44hgTJKIZuk2qN1kauF6thOw=
+			</data>
+		</dict>
+		<key>Headers/HuggingFaceTokenizer.h</key>
+		<dict>
+			<key>hash2</key>
+			<data>
+			0ETM5qw12+W8ULx2zP2UkVomFrBRnwAr8I7po2ACk/k=
 			</data>
 		</dict>
 		<key>Headers/LLaMARunner.h</key>
 		<dict>
 			<key>hash2</key>
 			<data>
-			or8gFkCO2QVkQgaeFAaqs/WqGjv8kABL8Rzcdcuexw0=
+			AZECdFqfkT4YWu8Nqga5ALaJUme8IOAnhOaAUM+iOvU=
 			</data>
 		</dict>
 	</dict>