chore: Update tokenizers (#823)

## Description

This PR adds new tokenizer headers and bumps executorch binaries to
reflect the changes.

### Introduces a breaking change?

- [ ] Yes
- [x] No

### Type of change

- [ ] Bug fix (change which fixes an issue)
- [ ] New feature (change which adds functionality)
- [ ] Documentation update (improves or adds clarity to existing
documentation)
- [x] Other (chores, tests, code style improvements etc.)

### Tested on

- [ ] iOS
- [ ] Android

### Testing instructions

Run apps for LLM, T2I, S2T, Embedding

Author: benITo47

packages/react-native-executorch/android/libs/classes.jar

@@ -80,4 +80,4 @@ class HFTokenizer : public Tokenizer {
   Model::Ptr _model;
 };
 
-} // namespace tokenizers
+} // namespace tokenizers

packages/react-native-executorch/third-party/android/libs/executorch/arm64-v8a/libexecutorch.so

@@ -148,4 +148,4 @@ class ModelConfig {
   Model::Ptr create() const;
 };
 
-} // namespace tokenizers
+} // namespace tokenizers

packages/react-native-executorch/third-party/android/libs/executorch/x86_64/libexecutorch.so

@@ -104,6 +104,11 @@ class PreTokenizerConfig {
    */
   PRETOKENIZER_CONFIG_MEMBER(bool, add_prefix_space)
 
+  /**
+   * Used by: ByteLevelPreTokenizer
+   */
+  PRETOKENIZER_CONFIG_MEMBER(bool, use_regex)
+
   /**
    * Used by RegexPreTokenizer
    */
@@ -225,11 +230,13 @@ class ByteLevelPreTokenizer : public PreTokenizer {
    * @param add_prefix_space: Whether to add a leading space to the first word
    * @param pattern: A user-supplied regex to use for token splitting. If not
    *    provided, it use the standard GPT2 pattern.
+   * @param use_regex: Whether to use regex for splitting. If false, only apply
+   *    byte encoding without splitting.
    */
   ByteLevelPreTokenizer(bool add_prefix_space = true,
-                        const std::string &pattern = "");
+                        const std::string &pattern = "", bool use_regex = true);
   explicit ByteLevelPreTokenizer(const std::string &pattern)
-      : ByteLevelPreTokenizer(true, pattern) {}
+      : ByteLevelPreTokenizer(true, pattern, true) {}
 
   /** Perform pre-tokenization */
   std::vector<std::string>
@@ -238,6 +245,7 @@ class ByteLevelPreTokenizer : public PreTokenizer {
 private:
   const std::string pattern_;
   const bool add_prefix_space_;
+  const bool use_regex_;
 
 }; // end class ByteLevelPreTokenizer
 

packages/react-native-executorch/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/hf_tokenizer.h

@@ -89,4 +89,4 @@ class TruncationConfig {
   TruncationParams params;
 };
 
-} // namespace tokenizers
+} // namespace tokenizers

packages/react-native-executorch/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/model.h

@@ -0,0 +1,48 @@
+/*
+ * Unicode NFC/NFD normalization implementation
+ * This implementation fixes the broken llama.cpp-unicode normalization
+ * by using proper multi-codepoint decomposition sequences.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace tokenizers {
+
+/**
+ * Normalize a sequence of codepoints to NFD (Canonical Decomposition)
+ *
+ * @param cpts Input codepoint sequence
+ * @return NFD-normalized codepoint sequence
+ */
+std::vector<uint32_t> unicode_normalize_nfd(const std::vector<uint32_t> &cpts);
+
+/**
+ * Normalize a sequence of codepoints to NFC (Canonical Decomposition +
+ * Composition)
+ *
+ * @param cpts Input codepoint sequence
+ * @return NFC-normalized codepoint sequence
+ */
+std::vector<uint32_t> unicode_normalize_nfc(const std::vector<uint32_t> &cpts);
+
+/**
+ * Normalize a UTF-8 string to NFD
+ *
+ * @param utf8 Input UTF-8 string
+ * @return NFD-normalized UTF-8 string
+ */
+std::string unicode_normalize_nfd_utf8(const std::string &utf8);
+
+/**
+ * Normalize a UTF-8 string to NFC
+ *
+ * @param utf8 Input UTF-8 string
+ * @return NFC-normalized UTF-8 string
+ */
+std::string unicode_normalize_nfc_utf8(const std::string &utf8);
+
+} // namespace tokenizers