Merge pull request #24 from huggingface/nico/regex-discrepancies-with-python-23

Improve robustness for handling Python-authored regexes

Author: nico-martin

.gitattributes

@@ -0,0 +1 @@
+*.gz binary

.prettierignore

@@ -0,0 +1,2 @@
+tests/py/*.py
+*.gz

README.md

@@ -64,6 +64,12 @@ This library expects two files from Hugging Face models:
 - `tokenizer.json` - Contains the tokenizer configuration
 - `tokenizer_config.json` - Contains additional metadata
 
+## Regex compatibility
+
+Tokenizer configs are authored for the Rust `tokenizers` crate, which compiles patterns with Oniguruma — a regex engine whose syntax and Unicode semantics differ from JavaScript `RegExp`. Tokenizers.js translates these patterns structurally, matching Oniguruma behavior for: line anchors (`^`/`$` recognize only `\n`, unlike JavaScript's `m` flag), absolute anchors (`\A`, `\z`, `\Z`), `.` (which excludes only `\n`), word/digit/space shorthands (including Oniguruma's exact word-character set and its Latin-1 ctype quirks), `\h`/`\H` hex digits, `\b`/`\B` boundaries, inline case-insensitive groups (including ranges like `(?i:[a-f])`), possessive and stacked quantifiers (`a++`, `X{3}+`), atomic groups, POSIX bracket expressions, `\x{...}` code points, `\p{Word}`, identity escapes, and literal braces/brackets.
+
+A few constructs can't be fully reproduced — notably `\G`, full Unicode case folding (e.g. `ß` ~ `ss`), character-class intersection (`&&`), and the `MergedWithPrevious`/`MergedWithNext`/`Contiguous` split behaviors.
+
 ## Components
 
 Tokenizers.js supports [Hugging Face tokenizer components](https://huggingface.co/docs/tokenizers/components):

jest.config.mjs

@@ -139,6 +139,10 @@ export default {
   // Options that will be passed to the testEnvironment
   // testEnvironmentOptions: {},
 
+  // Default timeout of a test/hook in milliseconds (raised from Jest's 5s default to allow
+  // for network fetches of tokenizer configs on a cold cache).
+  testTimeout: 15000,
+
   // Adds a location field to test results
   // testLocationInResults: false,
 

pnpm-workspace.yaml

@@ -1,3 +1,5 @@
 allowBuilds:
   esbuild: true
   unrs-resolver: true
+
+minimumReleaseAge: 1440

src/core/decoder/Replace.ts

@@ -4,9 +4,20 @@ import { TokenizerConfigDecoderReplace } from "@static/tokenizer";
 
 class Replace extends Decoder {
   declare config: TokenizerConfigDecoderReplace;
+  pattern: RegExp | null;
+
+  /**
+   * @param config The configuration object for the decoder.
+   */
+  constructor(config: TokenizerConfigDecoderReplace) {
+    super(config);
+    // Compile once: decode_chain() is called for every decode.
+    this.pattern = create_pattern(this.config.pattern);
+  }
+
   decode_chain(tokens: string[]): string[] {
-    const pattern = create_pattern(this.config.pattern);
     const content = this.config.content ?? "";
+    const pattern = this.pattern;
     return pattern === null
       ? tokens
       : tokens.map((token) => token.replaceAll(pattern, content));

src/core/normalizer/Replace.ts

@@ -8,17 +8,26 @@ import type { TokenizerConfigNormalizerReplace } from "@static/tokenizer";
  */
 class Replace extends Normalizer {
   declare config: TokenizerConfigNormalizerReplace;
+  pattern: RegExp | null;
+
+  /**
+   * @param config The configuration object for the normalizer.
+   */
+  constructor(config: TokenizerConfigNormalizerReplace) {
+    super(config);
+    // Compile once: normalize() is called for every input text.
+    this.pattern = create_pattern(this.config.pattern ?? {});
+  }
 
   /**
    * Normalize the input text by replacing the pattern with the content.
    * @param text The input text to be normalized.
    * @returns The normalized text after replacing the pattern with the content.
    */
   normalize(text: string): string {
-    const pattern = create_pattern(this.config.pattern ?? {});
-    return pattern === null
+    return this.pattern === null
       ? text
-      : text.replaceAll(pattern, this.config.content ?? "");
+      : text.replaceAll(this.pattern, this.config.content ?? "");
   }
 }
 

src/core/preTokenizer/Split.ts

@@ -35,7 +35,7 @@ class Split extends PreTokenizer {
     }
 
     if (this.config.invert) {
-      return text.match(this.pattern) || [];
+      return (text.match(this.pattern) || []).filter((x) => x);
     } else if (this.config.behavior?.toLowerCase() === "removed") {
       return text.split(this.pattern).filter((x) => x);
     } else {

src/static/constants.ts

@@ -35,39 +35,5 @@ const reverse_dictionary = (data: Object) =>
 
 export const UNICODE_TO_BYTES = reverse_dictionary(BYTES_TO_UNICODE);
 
-const BLOOM_SPLIT_CHARS = ".,!?\u2026\u3002\uff0c\u3001\u0964\u06d4\u060c";
-
-export const PROBLEMATIC_REGEX_MAP = new Map([
-  // These uses the case insensitive group modifier, which is not supported in JavaScript.
-  // When parsing the regex, an "Invalid group" error is thrown.
-  [
-    "(?i:'s|'t|'re|'ve|'m|'ll|'d)",
-    "(?:'([sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD]))",
-  ],
-  [
-    "(?i:[sdmt]|ll|ve|re)",
-    "(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])",
-  ],
-
-  // JS doesn't support possessive quantifiers (these are used in recent OpenAI tokenizers).
-  ["[^\\r\\n\\p{L}\\p{N}]?+", "[^\\r\\n\\p{L}\\p{N}]?"],
-  ["[^\\s\\p{L}\\p{N}]++", "[^\\s\\p{L}\\p{N}]+"],
-
-  // JS doesn't support atomic groups (these are used in AFMoE tokenizers).
-  ["(?>\\p{Nd}{510})", "(?:\\p{Nd}{510})"],
-
-  // JS doesn't support stacking quantifiers.
-  // Uncaught SyntaxError: Invalid regular expression: /\p{Nd}{3}+/u: Nothing to repeat
-  ["\\p{Nd}{3}+", "(?:\\p{Nd}{3})+"],
-
-  // \G is an invalid escape in JS, and in most cases is just used as an optimization.
-  // So, we can safely remove it.
-  ["\\G", ""],
-
-  // Used to override the default (invalid) regex of the bloom pretokenizer.
-  // For more information, see https://github.com/huggingface/transformers.js/issues/94
-  [` ?[^(\\s|[${BLOOM_SPLIT_CHARS}])]+`, ` ?[^\\s${BLOOM_SPLIT_CHARS}]+`],
-]);
-
 export const PUNCTUATION_REGEX =
   "\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E";