fix: handle NoneType check for think tokens in TokenizerWrapper (#1167)

Author: yuetyeelo2855

mlx_lm/tokenizer_utils.py

@@ -397,6 +397,8 @@ def think_start(self):
 
     @property
     def think_start_id(self):
+        if self._think_start_tokens is None:
+            return None
         if len(self._think_start_tokens) > 1:
             raise ValueError("The start thinking sequence is more than 1 token")
         return self._think_start_tokens[0]
@@ -411,6 +413,8 @@ def think_end(self):
 
     @property
     def think_end_id(self):
+        if self._think_end_tokens is None:
+            return None
         if len(self._think_end_tokens) > 1:
             raise ValueError("The end thinking sequence is more than 1 token")
         return self._think_end_tokens[0]

tests/test_tokenizers.py

@@ -101,6 +101,14 @@ def test_thinking(self):
         self.assertEqual(tokenizer.think_start, "<think>")
         self.assertEqual(tokenizer.think_end, "</think>")
 
+        tokenizer_repo = "mlx-community/Llama-3.2-1B-Instruct-4bit"
+        tokenizer = load_tokenizer(tokenizer_repo)
+        self.assertFalse(tokenizer.has_thinking)
+        self.assertIsNone(tokenizer.think_start)
+        self.assertIsNone(tokenizer.think_end)
+        self.assertIsNone(tokenizer.think_start_id)
+        self.assertIsNone(tokenizer.think_end_id)
+
 
 if __name__ == "__main__":
     unittest.main()