Fix all ruff lint errors in tests/ and scripts/

- I001: Fix import ordering in test files and convert_hf_tokenizer.py
- C401: Replace generator with set comprehension
- B023: Replace lambda with direct method reference (parts.append)
- E501: Replace long pattern strings with PAT_GPT2 constant
- F401: Remove unused imports (regex, save_model in test_trainer.py)
- F841: Remove unused variable assignment

Author: neluca

scripts/convert_hf_tokenizer.py

@@ -21,7 +21,6 @@
 import sys
 from pathlib import Path
 
-
 # ---------------------------------------------------------------------------
 # GPT-2 ByteLevel mapping: bytes (0-255) ↔ visible Unicode characters
 # ---------------------------------------------------------------------------
@@ -205,7 +204,7 @@ def _detect_byte_mapping(vocab: dict[str, int]) -> list[int] | None:
     still_missing = [b for b in range(256) if bytes_maps[b] < 0]
     if still_missing:
         # Collect all single-char tokens not yet assigned
-        assigned_ids = set(bytes_maps[b] for b in range(256) if bytes_maps[b] >= 0)
+        assigned_ids = {bytes_maps[b] for b in range(256) if bytes_maps[b] >= 0}
         unassigned = [
             (ord(c), tid)
             for c, tid in char_to_id.items()

tests/test_fuzz.py

@@ -93,7 +93,7 @@ def test_stream_decode_random(self):
 
             # Streaming decode
             parts: list[str] = []
-            decoder = tok.stream_decode(lambda s: parts.append(s))
+            decoder = tok.stream_decode(parts.append)
             for tid in ids:
                 decoder(tid)
             streamed = "".join(parts)

tests/test_hf_models.py

@@ -119,7 +119,7 @@ def test_streaming_decode(self, model_key, hf_id, desc, pattern):
                 continue
             ids = tok.encode(text)
             parts: list[str] = []
-            decoder = tok.stream_decode(lambda s: parts.append(s))
+            decoder = tok.stream_decode(parts.append)
             for tid in ids:
                 decoder(tid)
             assert "".join(parts) == text
@@ -165,8 +165,7 @@ class TestQwen25Specific:
 
     def test_chinese_text(self):
         """Chinese text should encode and decode correctly."""
-        pattern = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
-        tok = Tokenizer.from_file("models/qwen25.tbm", pat_str=pattern)
+        tok = Tokenizer.from_file("models/qwen25.tbm", pat_str=PAT_GPT2)
 
         cn_texts = [
             "他是一个独自一人划着小船在墨西哥湾大海流打鱼的老人",
@@ -181,8 +180,7 @@ def test_chinese_text(self):
 
     def test_code_text(self):
         """Code snippets should round-trip."""
-        pattern = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
-        tok = Tokenizer.from_file("models/qwen25.tbm", pat_str=pattern)
+        tok = Tokenizer.from_file("models/qwen25.tbm", pat_str=PAT_GPT2)
 
         code = '''def fibonacci(n):
     if n <= 1:
@@ -199,8 +197,7 @@ class TestPhi2Specific:
 
     def test_code_roundtrip(self):
         """Code snippets should round-trip with Phi-2."""
-        pattern = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
-        tok = Tokenizer.from_file("models/phi2.tbm", pat_str=pattern)
+        tok = Tokenizer.from_file("models/phi2.tbm", pat_str=PAT_GPT2)
 
         code = "def hello(): return 'world'"
         ids = tok.encode(code)
@@ -209,8 +206,7 @@ def test_code_roundtrip(self):
 
     def test_english_text(self):
         """English text should round-trip accurately."""
-        pattern = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
-        tok = Tokenizer.from_file("models/phi2.tbm", pat_str=pattern)
+        tok = Tokenizer.from_file("models/phi2.tbm", pat_str=PAT_GPT2)
 
         text = "The quick brown fox jumps over the lazy dog."
         ids = tok.encode(text)

tests/test_models.py

@@ -1,15 +1,16 @@
 """Tests for pre-built .tbm models — verify correctness against tiktoken."""
 
 import pytest
+
 from tinybpe import Tokenizer
 
 # Try importing tiktoken — tests are skipped if not available
 try:
     import tiktoken
 
-    HAS_TIKTOKEN = True
+    HAS_TIKTOKEN = True  # noqa: F401
 except ImportError:
-    HAS_TIKTOKEN = False
+    HAS_TIKTOKEN = False  # noqa: F401
 
 
 # ---------------------------------------------------------------------------
@@ -149,7 +150,7 @@ def test_streaming_decode(self, name, desc):
                 continue
             ids = tok.encode(text)
             parts: list[str] = []
-            decoder = tok.stream_decode(lambda s: parts.append(s))
+            decoder = tok.stream_decode(parts.append)
             for tid in ids:
                 decoder(tid)
             assert "".join(parts) == text

tests/test_tokenizer.py

@@ -2,7 +2,7 @@
 
 from pathlib import Path
 
-from tinybpe import Tokenizer, load_model, save_model, save_vocab, load_vocab
+from tinybpe import Tokenizer, load_model, load_vocab, save_model, save_vocab
 
 TESTS_DIR = Path(__file__).parent
 FILE_SIMPLE = str(TESTS_DIR / "simple")

tests/test_trainer.py

@@ -1,9 +1,9 @@
 """Tests for the TinyBPE Trainer."""
 
-import regex as re
-from tinybpe import Trainer, Tokenizer, save_model, load_model
 from pathlib import Path
 
+from tinybpe import Tokenizer, Trainer, load_model
+
 TESTS_DIR = Path(__file__).parent
 
 
@@ -112,7 +112,7 @@ class TestTrainerEdgeCases:
     def test_empty_text(self):
         """Empty text should produce a valid but idle trainer."""
         try:
-            trainer = Trainer("")
+            Trainer("")  # should raise ValueError
         except ValueError:
             # Expected: empty list not allowed
             pass