Simplify added-token prefix-space fix

Fold the fix into `_get_prepend_scheme`: it now returns "first" instead
of "always" when `add_prefix_space=True`, so standalone Metaspace
pre_tokenizers minted via `convert_slow_tokenizer` no longer leave a
spurious "▁" prefix on the chunk after an added token (#28218). "first"
and "always" are identical on a standalone Metaspace for text without
added tokens.

WhitespaceSplit + Metaspace genuinely needs "always" so each split word
keeps its "▁" SentencePiece marker; PegasusConverter and XLNetTokenizer
now hardcode that scheme.

Drop the `_init_complete` guard and `_align_metaspace_for_added_tokens`
helper: the swap to "first" is inlined in `_add_tokens` and triggers
when a non-special token is registered (covers .json loads that still
bake in "always", whether the token comes from persisted state or a
user `add_tokens` call).

BigBird's persisted `<sep_*>`/`<length_*>` slots are non-special, so
the swap now runs at load time and the chunk after a literal `<s>`
loses its prefix; integration fixtures updated to match.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: qflen

src/transformers/convert_slow_tokenizer.py

@@ -110,13 +110,8 @@ def import_protobuf(error_message=""):
 
 
 def _get_prepend_scheme(add_prefix_space: bool, original_tokenizer) -> str:
-    if add_prefix_space:
-        prepend_scheme = "always"
-        if not getattr(original_tokenizer, "legacy", True):
-            prepend_scheme = "first"
-    else:
-        prepend_scheme = "never"
-    return prepend_scheme
+    # "first" avoids the spurious "▁" prefix on chunks following a user-added token (#28218).
+    return "first" if add_prefix_space else "never"
 
 
 def generate_merges(vocab, vocab_scores, skip_tokens: Collection[str] | None = None):
@@ -1316,7 +1311,8 @@ def unk_id(self, proto):
         return proto.trainer_spec.unk_id + self.original_tokenizer.offset
 
     def pre_tokenizer(self, replacement, add_prefix_space):
-        prepend_scheme = _get_prepend_scheme(add_prefix_space, self.original_tokenizer)
+        # WhitespaceSplit + Metaspace needs "always" so each split word gets the "▁" marker.
+        prepend_scheme = "always" if add_prefix_space else "never"
         return pre_tokenizers.Sequence(
             [
                 pre_tokenizers.WhitespaceSplit(),

src/transformers/models/xlnet/tokenization_xlnet.py

@@ -16,7 +16,6 @@
 from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, processors
 from tokenizers.models import Unigram
 
-from ...tokenization_utils_base import _get_prepend_scheme
 from ...tokenization_utils_tokenizers import TokenizersBackend
 from ...utils import logging
 
@@ -144,7 +143,8 @@ def __init__(
         self._tokenizer.normalizer = normalizers.Sequence(list_normalizers)
 
         add_prefix_space = True
-        prepend_scheme = _get_prepend_scheme(add_prefix_space, self)
+        # WhitespaceSplit + Metaspace needs "always" so each split word gets the "▁" marker.
+        prepend_scheme = "always" if add_prefix_space else "never"
         self._tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
             [
                 pre_tokenizers.WhitespaceSplit(),

src/transformers/tokenization_utils_base.py

@@ -3547,13 +3547,8 @@ def load_vocab_and_merges(pretrained_model_name_or_path, **kwargs):
 
 
 def _get_prepend_scheme(add_prefix_space: bool, original_tokenizer) -> str:
-    if add_prefix_space:
-        prepend_scheme = "always"
-        if not getattr(original_tokenizer, "legacy", True):
-            prepend_scheme = "first"
-    else:
-        prepend_scheme = "never"
-    return prepend_scheme
+    # "first" avoids the spurious "▁" prefix on chunks following a user-added token (#28218).
+    return "first" if add_prefix_space else "never"
 
 
 def generate_merges(vocab, vocab_scores: dict[str, float] | None = None, skip_tokens: Collection[str] | None = None):

src/transformers/tokenization_utils_tokenizers.py

@@ -488,8 +488,6 @@ def __init__(self, *args, **kwargs):
         if self._should_update_post_processor:
             self.update_post_processor()
 
-        self._init_complete = True
-
     @property
     def is_fast(self) -> bool:
         return True
@@ -729,24 +727,19 @@ def _add_tokens(self, new_tokens: list[str | AddedToken], special_tokens=False)
         if special_tokens:
             return self._tokenizer.add_special_tokens(new_tokens)
 
-        if getattr(self, "_init_complete", False) and any(isinstance(t, str) or not t.special for t in new_tokens):
-            self._align_metaspace_for_added_tokens()
+        # For tokenizers loaded from a .json that bakes in prepend_scheme="always",
+        # swap a standalone Metaspace to "first" once a non-special token enters so
+        # the chunk after the added token does not pick up a spurious "▁" (#28218).
+        if any(isinstance(t, str) or not t.special for t in new_tokens):
+            pre_tokenizer = self._tokenizer.pre_tokenizer
+            if isinstance(pre_tokenizer, pre_tokenizers_fast.Metaspace) and pre_tokenizer.prepend_scheme == "always":
+                self._tokenizer.pre_tokenizer = pre_tokenizers_fast.Metaspace(
+                    replacement=pre_tokenizer.replacement,
+                    prepend_scheme="first",
+                    split=pre_tokenizer.split,
+                )
         return self._tokenizer.add_tokens(new_tokens)
 
-    def _align_metaspace_for_added_tokens(self) -> None:
-        # Mirror `SpmConverter.pre_tokenizer` for non-legacy spm tokenizers: switch a
-        # standalone Metaspace pre_tokenizer from prepend_scheme="always" to "first"
-        # so chunks following a user-added token are not given a spurious "▁" prefix.
-        # Sequence pipelines (eg. WhitespaceSplit + Metaspace) rely on "always"
-        # semantics for whitespace handling and are intentionally skipped.
-        pre_tokenizer = self._tokenizer.pre_tokenizer
-        if isinstance(pre_tokenizer, pre_tokenizers_fast.Metaspace) and pre_tokenizer.prepend_scheme == "always":
-            self._tokenizer.pre_tokenizer = pre_tokenizers_fast.Metaspace(
-                replacement=pre_tokenizer.replacement,
-                prepend_scheme="first",
-                split=pre_tokenizer.split,
-            )
-
     def num_special_tokens_to_add(self, pair: bool = False) -> int:
         """
         Returns the number of added tokens when encoding a sequence with special tokens.

tests/models/big_bird/test_tokenization_big_bird.py

@@ -31,7 +31,7 @@ class BigBirdTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
     from_pretrained_id = "google/bigbird-roberta-base"
     tokenizer_class = BigBirdTokenizer
 
-    integration_expected_tokens = ['▁This', '▁is', '▁a', '▁test', '▁', '😊\n', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fals', 'é', '.', '\n生活的真谛是\n', 'Hi', '▁Hello', '\n', 'Hi', '▁Hello', '\n\n', '▁', '\n', '▁', '\n', '▁Hello', '\n', '<s>', '▁', '\n', 'hi', '<s>', '▁there', '\n', 'The', '▁following', '▁string', '▁should', '▁be', '▁properly', '▁encoded', ':', '▁Hello', '.', '\n', 'But', '▁', 'ird', '▁and', '▁', 'ปี', '▁', 'ird', '▁', 'ด\n', 'Hey', '▁how', '▁are', '▁you', '▁doing']  # fmt: skip
-    integration_expected_token_ids = [871, 419, 358, 1433, 321, 100, 141, 474, 4743, 388, 961, 11125, 112, 391, 529, 419, 27908, 266, 114, 100, 17351, 18536, 100, 17351, 18536, 100, 321, 100, 321, 100, 18536, 100, 2, 321, 100, 5404, 2, 713, 100, 565, 1809, 4832, 916, 408, 6206, 30341, 126, 18536, 114, 100, 1638, 321, 1548, 391, 321, 100, 321, 1548, 321, 100, 10915, 804, 490, 446, 1905]  # fmt: skip
-    expected_tokens_from_ids = ['▁This', '▁is', '▁a', '▁test', '▁', '<unk>', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fals', 'é', '.', '<unk>', 'Hi', '▁Hello', '<unk>', 'Hi', '▁Hello', '<unk>', '▁', '<unk>', '▁', '<unk>', '▁Hello', '<unk>', '<s>', '▁', '<unk>', 'hi', '<s>', '▁there', '<unk>', 'The', '▁following', '▁string', '▁should', '▁be', '▁properly', '▁encoded', ':', '▁Hello', '.', '<unk>', 'But', '▁', 'ird', '▁and', '▁', '<unk>', '▁', 'ird', '▁', '<unk>', 'Hey', '▁how', '▁are', '▁you', '▁doing']  # fmt: skip
-    integration_expected_decoded_text = "This is a test <unk>I was born in 92000, and this is falsé.<unk>Hi Hello<unk>Hi Hello<unk> <unk> <unk> Hello<unk><s> <unk>hi<s> there<unk>The following string should be properly encoded: Hello.<unk>But ird and <unk> ird <unk>Hey how are you doing"
+    integration_expected_tokens = ['▁This', '▁is', '▁a', '▁test', '▁', '😊\n', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fals', 'é', '.', '\n生活的真谛是\n', 'Hi', '▁Hello', '\n', 'Hi', '▁Hello', '\n\n', '▁', '\n', '▁', '\n', '▁Hello', '\n', '<s>', '\n', 'hi', '<s>', 'there', '\n', 'The', '▁following', '▁string', '▁should', '▁be', '▁properly', '▁encoded', ':', '▁Hello', '.', '\n', 'But', '▁', 'ird', '▁and', '▁', 'ปี', '▁', 'ird', '▁', 'ด\n', 'Hey', '▁how', '▁are', '▁you', '▁doing']  # fmt: skip
+    integration_expected_token_ids = [871, 419, 358, 1433, 321, 100, 141, 474, 4743, 388, 961, 11125, 112, 391, 529, 419, 27908, 266, 114, 100, 17351, 18536, 100, 17351, 18536, 100, 321, 100, 321, 100, 18536, 100, 2, 100, 5404, 2, 8218, 100, 565, 1809, 4832, 916, 408, 6206, 30341, 126, 18536, 114, 100, 1638, 321, 1548, 391, 321, 100, 321, 1548, 321, 100, 10915, 804, 490, 446, 1905]  # fmt: skip
+    expected_tokens_from_ids = ['▁This', '▁is', '▁a', '▁test', '▁', '<unk>', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fals', 'é', '.', '<unk>', 'Hi', '▁Hello', '<unk>', 'Hi', '▁Hello', '<unk>', '▁', '<unk>', '▁', '<unk>', '▁Hello', '<unk>', '<s>', '<unk>', 'hi', '<s>', 'there', '<unk>', 'The', '▁following', '▁string', '▁should', '▁be', '▁properly', '▁encoded', ':', '▁Hello', '.', '<unk>', 'But', '▁', 'ird', '▁and', '▁', '<unk>', '▁', 'ird', '▁', '<unk>', 'Hey', '▁how', '▁are', '▁you', '▁doing']  # fmt: skip
+    integration_expected_decoded_text = "This is a test <unk>I was born in 92000, and this is falsé.<unk>Hi Hello<unk>Hi Hello<unk> <unk> <unk> Hello<unk><s><unk>hi<s>there<unk>The following string should be properly encoded: Hello.<unk>But ird and <unk> ird <unk>Hey how are you doing"

tests/tokenization/test_tokenization_fast.py

@@ -218,9 +218,8 @@ def test_class_after_save_and_reload(self):
             self.assertIsInstance(tokenizer, PreTrainedTokenizerFast)
 
     def test_added_token_does_not_get_metaspace_prefix(self):
-        # Issue #28218: when a non-special token is added to a SentencePiece-style fast
-        # tokenizer, the chunk that follows the added token must not be given a
-        # spurious "▁" prefix from the Metaspace pre_tokenizer.
+        # #28218: a non-special add_tokens call on a SentencePiece-style fast tokenizer
+        # must not leave a spurious "▁" prefix on the chunk after the added token.
         vocab = [
             ("<unk>", 0.0),
             ("▁", -10.0),