Scope prepend_scheme realignment to post-init add_tokens

The `_add_tokens` override flipped a standalone Metaspace("always") to
"first" on every non-special add, including a tokenizer's own added
tokens restored during `from_pretrained`. For a checkpoint that loads
as a standalone Metaspace with non-special added tokens (e.g. kosmos-2,
which restores 1035 of them), this rewrote the saved scheme at load
time and dropped the metaspace prefix after in-text special tokens,
breaking Kosmos2ProcessorTest::test_full_processor.

Gate the flip on a `_prepend_scheme_can_flip` flag set once construction
finishes, so only genuine post-init user `add_tokens` calls realign the
scheme. Add a regression test covering checkpoint restore.

Author: qflen

src/transformers/tokenization_utils_tokenizers.py

@@ -488,6 +488,10 @@ def __init__(self, *args, **kwargs):
         if self._should_update_post_processor:
             self.update_post_processor()
 
+        # Construction (incl. checkpoint-restore of added tokens) is done; later non-special adds are
+        # genuine user edits and may realign the Metaspace prepend_scheme in `_add_tokens` (#28218).
+        self._prepend_scheme_can_flip = True
+
     @property
     def is_fast(self) -> bool:
         return True
@@ -727,10 +731,12 @@ def _add_tokens(self, new_tokens: list[str | AddedToken], special_tokens=False)
         if special_tokens:
             return self._tokenizer.add_special_tokens(new_tokens)
 
-        # For tokenizers loaded from a .json that bakes in prepend_scheme="always",
-        # swap a standalone Metaspace to "first" once a non-special token enters so
-        # the chunk after the added token does not pick up a spurious "▁" (#28218).
-        if any(isinstance(t, str) or not t.special for t in new_tokens):
+        # #28218: a non-special token added to a standalone Metaspace(prepend_scheme="always") leaves a
+        # spurious "▁" on the chunk after it; realign to "first" so only the first chunk is prefixed. Gated
+        # on post-init so restoring a checkpoint's own added tokens does not silently change its scheme.
+        if getattr(self, "_prepend_scheme_can_flip", False) and any(
+            isinstance(t, str) or not t.special for t in new_tokens
+        ):
             pre_tokenizer = self._tokenizer.pre_tokenizer
             if isinstance(pre_tokenizer, pre_tokenizers_fast.Metaspace) and pre_tokenizer.prepend_scheme == "always":
                 self._tokenizer.pre_tokenizer = pre_tokenizers_fast.Metaspace(

tests/tokenization/test_tokenization_fast.py

@@ -243,6 +243,27 @@ def test_added_token_does_not_get_metaspace_prefix(self):
             ["▁Hi", "▁", "abcd", "gym", "▁world"],
         )
 
+    def test_checkpoint_restore_keeps_metaspace_prepend_scheme(self):
+        # #28218 follow-up: restoring a checkpoint that legitimately baked prepend_scheme="always"
+        # together with its own non-special added tokens must NOT flip the scheme to "first". The
+        # realignment only applies to genuine post-init user add_tokens, never to load-time restore.
+        vocab = [("<unk>", 0.0), ("▁", -10.0), ("gym", 0.0), ("▁gym", 0.0)]
+        backend = Tokenizer(Unigram(vocab=vocab, unk_id=0))
+        backend.pre_tokenizer = pre_tokenizers.Metaspace(replacement="▁", prepend_scheme="always", split=True)
+        backend.add_tokens(["abcd"])  # a non-special token baked into the checkpoint
+        fast = PreTrainedTokenizerFast(tokenizer_object=backend, unk_token="<unk>")
+        self.assertEqual(fast._tokenizer.pre_tokenizer.prepend_scheme, "always")
+
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            fast.save_pretrained(tmp_dir)
+            reloaded = PreTrainedTokenizerFast.from_pretrained(tmp_dir)
+
+        # Restoring "abcd" while loading must leave the scheme untouched...
+        self.assertEqual(reloaded._tokenizer.pre_tokenizer.prepend_scheme, "always")
+        # ...while a genuine post-load user add still realigns it.
+        reloaded.add_tokens(["efgh"])
+        self.assertEqual(reloaded._tokenizer.pre_tokenizer.prepend_scheme, "first")
+
     def test_bpe_tokenizer_skips_clean_up_tokenization_spaces(self):
         """BPE tokenizers should not apply clean_up_tokenization even when the flag is True.