Fix AutoTokenizer.register() for transformers 5.13.0+ compatibility

mlx_lm/tokenizer_utils.py

@@ -5,7 +5,7 @@
 from json import JSONDecodeError
 from typing import Any, Dict, List, Optional
 
-from transformers import AutoTokenizer, PreTrainedTokenizerFast
+from transformers import AutoTokenizer, PreTrainedConfig, PreTrainedTokenizerFast
 
 
 class StreamingDetokenizer:
@@ -474,6 +474,15 @@ def __setattr__(self, attr, value):
             setattr(self._tokenizer, attr, value)
 
 
+class NewlineTokenizerConfig(PreTrainedConfig):
+    """Configuration for NewlineTokenizer."""
+
+    model_type = "newline_tokenizer"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+
 class NewlineTokenizer(PreTrainedTokenizerFast):
     """A tokenizer that replaces newlines with <n> and <n> with new line."""
 
@@ -500,7 +509,7 @@ def batch_decode(self, *args, **kwargs):
         return [self._postprocess_text(d) for d in decoded]
 
 
-AutoTokenizer.register("NewlineTokenizer", fast_tokenizer_class=NewlineTokenizer)
+AutoTokenizer.register(NewlineTokenizerConfig, fast_tokenizer_class=NewlineTokenizer)
 
 
 def _match(a, b):

tests/test_tokenizers.py

@@ -8,6 +8,8 @@
 from mlx_lm.tokenizer_utils import (
     BPEStreamingDetokenizer,
     NaiveStreamingDetokenizer,
+    NewlineTokenizer,
+    NewlineTokenizerConfig,
     SPMStreamingDetokenizer,
     TokenizerWrapper,
 )
@@ -123,6 +125,33 @@ def test_find_token(self):
         self.assertEqual(find(prompt, [THINK_START], start=0), 1)
         self.assertEqual(find(prompt, [THINK_START], start=0, reverse=True), 3)
 
+    def test_newline_tokenizer_registration(self):
+        """Test that NewlineTokenizer is properly registered with a config class.
+
+        This test verifies the fix for transformers 5.13.0+ compatibility where
+        AutoTokenizer.register() requires a config class instead of a string.
+        """
+        from transformers import AutoTokenizer
+
+        # Verify the config class exists and has required attributes
+        self.assertTrue(hasattr(NewlineTokenizerConfig, "model_type"))
+        self.assertEqual(NewlineTokenizerConfig.model_type, "newline_tokenizer")
+
+        # Verify the config class has __module__ attribute (required by transformers 5.13.0+)
+        self.assertTrue(hasattr(NewlineTokenizerConfig, "__module__"))
+        self.assertFalse(NewlineTokenizerConfig.__module__.startswith("transformers."))
+
+        # Verify NewlineTokenizer class exists and is properly defined
+        self.assertTrue(
+            issubclass(NewlineTokenizer, AutoTokenizer.__bases__[0].__class__)
+        )
+
+        # Test that the tokenizer can process newlines correctly
+        # Note: We can't easily test the full registration without a tokenizer file,
+        # but we can verify the class methods work as expected
+        self.assertTrue(hasattr(NewlineTokenizer, "_preprocess_text"))
+        self.assertTrue(hasattr(NewlineTokenizer, "_postprocess_text"))
+
 
 if __name__ == "__main__":
     unittest.main()