docstrings, address reviewer comments

Author: stephantul

model2vec/distill/distillation.py

@@ -80,7 +80,7 @@ def distill_from_model(
 
     # Clean the vocabulary by removing duplicate tokens and tokens that are in the internal vocabulary.
     # Copy the original tokenizer model.
-    tokenizer_model = original_tokenizer_model._deep_copy()
+    tokenizer_model = original_tokenizer_model.model_copy(deep=True)
     if tokenizer_model.adds_prefix_space is not None:
         tokenizer_model.adds_prefix_space = True
 
@@ -184,7 +184,7 @@ def _validate_parameters(
         try:
             token_remove_regex = re.compile(token_remove_pattern)
         except re.error as e:
-            raise ValueError(f"Couldn't compile the regex pattern: {e}")
+            raise ValueError(f"Couldn't compile the regex pattern: {e}") from e
 
     return sif_coefficient, token_remove_regex
 

model2vec/tokenizer/__init__.py

@@ -2,7 +2,7 @@
 
 importable("transformers", "tokenizer")
 
-from model2vec.tokenizer.tokenizer import (
+from model2vec.tokenizer.tokenizer import (  # noqa: E402
     clean_and_create_vocabulary,
     turn_tokens_into_ids,
 )

model2vec/tokenizer/tokenizer.py

@@ -13,7 +13,7 @@ def clean_and_create_vocabulary(
     vocabulary_to_add: list[str],
     token_remove_regex: re.Pattern[str] | None,
 ) -> TokenizerModel:
-    """Cleans a vocabulary by removing duplicates and tokens that were already in the vocabulary."""
+    """Clean a vocabulary by removing duplicates and tokens that were already in the vocabulary."""
     seen_tokens = set()
 
     n_duplicate = 0
@@ -22,10 +22,9 @@ def clean_and_create_vocabulary(
 
     internal_tokens: list[str] = model.sorted_vocabulary
     if token_remove_regex:
-        len_before = len(internal_tokens)
         tokens_to_remove = [token for token in internal_tokens if token_remove_regex.match(token)]
         model = model.remove_tokens_from_vocabulary(tokens_to_remove)
-        n_regex_removed = len_before - len(internal_tokens)
+        n_regex_removed = len(tokens_to_remove)
     preprocessor = model.preprocessor
 
     seen_tokens = set(internal_tokens)
@@ -38,7 +37,7 @@ def clean_and_create_vocabulary(
             n_empty += 1
             continue
         if len(preprocessed) > 1:
-            tokens_as_str = [f"'{subword}'" for subword in token]
+            tokens_as_str = [f"'{subword}'" for subword in preprocessed]
             split_into = ",".join(tokens_as_str)
             logger.warning(f"Token '{token}' was split into multiple tokens after preprocessing: [{split_into}]")
             added_tokens_to_add.append(token)
@@ -65,7 +64,7 @@ def clean_and_create_vocabulary(
 
 
 def _report_statistics(n_multiword: int, n_duplicate: int, n_regex_removed: int, n_empty: int) -> None:
-    """Helper function to avoid increasing complexity in main function."""
+    """Report statistics on the various types of issues we found."""
     if n_multiword:
         logger.info(f"Added {n_multiword} multi-word tokens to the vocabulary.")
     if n_duplicate: