Format: ruff format on convert_tiktoken.py and tokenizer.py

Author: neluca

scripts/convert_tiktoken.py

@@ -41,7 +41,7 @@ def _decompose_token(mergeable_ranks: dict[bytes, int], token: bytes) -> list[by
         if min_idx is None:
             raise ValueError(f"Failed to find merge pair for token {token!r}")
 
-        parts = [*parts[:min_idx], parts[min_idx] + parts[min_idx + 1], *parts[min_idx + 2:]]
+        parts = [*parts[:min_idx], parts[min_idx] + parts[min_idx + 1], *parts[min_idx + 2 :]]
 
     return parts
 

tinybpe/tokenizer.py

@@ -125,9 +125,9 @@ def __init__(
             self._special_tokens = special_tokens
             # Sort by length descending so that longer tokens match before
             # shorter prefixes (e.g. "<ab>" before "<a>").
-            self._special_pattern = "(" + "|".join(
-                re.escape(k) for k in sorted(special_tokens, key=len, reverse=True)
-            ) + ")"
+            self._special_pattern = (
+                "(" + "|".join(re.escape(k) for k in sorted(special_tokens, key=len, reverse=True)) + ")"
+            )
 
             if self._bytes_maps is None:
                 _mapped = {k.encode("utf-8"): v for k, v in special_tokens.items()}