From b08c1a83fe353e8e63edf1d77057439c083fa668 Mon Sep 17 00:00:00 2001 From: qflen Date: Thu, 7 May 2026 12:06:29 +0200 Subject: [PATCH 1/3] Fix added-token prefix space in SentencePiece fast tokenizers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #28218. When a user calls add_tokens on a SentencePiece-style fast tokenizer (NLLB, Reformer, MLuke, ...), the tokens-trie splits the input at the new token and runs the Metaspace pre_tokenizer on each chunk. With the default prepend_scheme="always", every chunk is given a leading "▁", so "abcdgym" tokenizes to ["abcd", "▁gym"] and decodes back to "abcd gym". Mirror what SpmConverter.pre_tokenizer already does for non-legacy spm tokenizers: in TokenizersBackend._add_tokens, when a post-init, non-special add_tokens call lands on a standalone Metaspace pre_tokenizer with prepend_scheme="always", swap it for one with prepend_scheme="first" so only the first chunk receives the prefix. Sequence pipelines (WhitespaceSplit + Metaspace) are left untouched because they depend on "always" semantics for whitespace handling. Init-time additions of persisted tokens (e.g. BigBird's slots) are excluded via an _init_complete guard so existing integration tests for those tokenizers stay green. --- .../tokenization_utils_tokenizers.py | 18 ++++++++++++ tests/tokenization/test_tokenization_fast.py | 29 ++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/transformers/tokenization_utils_tokenizers.py b/src/transformers/tokenization_utils_tokenizers.py index 5365e9c1b6e1..6dbfa13e3e28 100644 --- a/src/transformers/tokenization_utils_tokenizers.py +++ b/src/transformers/tokenization_utils_tokenizers.py @@ -488,6 +488,8 @@ def __init__(self, *args, **kwargs): if self._should_update_post_processor: self.update_post_processor() + self._init_complete = True + @property def is_fast(self) -> bool: return True @@ -727,8 +729,24 @@ def _add_tokens(self, new_tokens: list[str | AddedToken], special_tokens=False) if special_tokens: return self._tokenizer.add_special_tokens(new_tokens) + if getattr(self, "_init_complete", False) and any(isinstance(t, str) or not t.special for t in new_tokens): + self._align_metaspace_for_added_tokens() return self._tokenizer.add_tokens(new_tokens) + def _align_metaspace_for_added_tokens(self) -> None: + # Mirror `SpmConverter.pre_tokenizer` for non-legacy spm tokenizers: switch a + # standalone Metaspace pre_tokenizer from prepend_scheme="always" to "first" + # so chunks following a user-added token are not given a spurious "▁" prefix + # (issue #28218). Sequence pipelines (eg. WhitespaceSplit + Metaspace) rely on + # "always" semantics for whitespace handling and are intentionally skipped. + pre_tokenizer = self._tokenizer.pre_tokenizer + if isinstance(pre_tokenizer, pre_tokenizers_fast.Metaspace) and pre_tokenizer.prepend_scheme == "always": + self._tokenizer.pre_tokenizer = pre_tokenizers_fast.Metaspace( + replacement=pre_tokenizer.replacement, + prepend_scheme="first", + split=pre_tokenizer.split, + ) + def num_special_tokens_to_add(self, pair: bool = False) -> int: """ Returns the number of added tokens when encoding a sequence with special tokens. diff --git a/tests/tokenization/test_tokenization_fast.py b/tests/tokenization/test_tokenization_fast.py index 68ca392982ee..d704d1e165a7 100644 --- a/tests/tokenization/test_tokenization_fast.py +++ b/tests/tokenization/test_tokenization_fast.py @@ -20,7 +20,7 @@ import unittest from tokenizers import Tokenizer, decoders, pre_tokenizers, trainers -from tokenizers.models import BPE, WordLevel +from tokenizers.models import BPE, Unigram, WordLevel from transformers import AutoTokenizer, PreTrainedTokenizerFast from transformers.testing_utils import require_tokenizers @@ -217,6 +217,33 @@ def test_class_after_save_and_reload(self): tokenizer = AutoTokenizer.from_pretrained(temp_dir, use_fast=True) self.assertIsInstance(tokenizer, PreTrainedTokenizerFast) + def test_added_token_does_not_get_metaspace_prefix(self): + # Issue #28218: when a non-special token is added to a SentencePiece-style fast + # tokenizer, the chunk that follows the added token must not be given a + # spurious "▁" prefix from the Metaspace pre_tokenizer. + vocab = [ + ("", 0.0), + ("▁", -10.0), + ("gym", 0.0), + ("▁gym", 0.0), + ("▁Hi", 0.0), + ("▁world", 0.0), + ] + backend = Tokenizer(Unigram(vocab=vocab, unk_id=0)) + backend.pre_tokenizer = pre_tokenizers.Metaspace(replacement="▁", prepend_scheme="always", split=True) + fast = PreTrainedTokenizerFast(tokenizer_object=backend, unk_token="") + self.assertEqual(fast._tokenizer.pre_tokenizer.prepend_scheme, "always") + + fast.add_tokens(["abcd"]) + + self.assertEqual(fast._tokenizer.pre_tokenizer.prepend_scheme, "first") + self.assertEqual(fast.tokenize("abcdgym"), ["abcd", "gym"]) + self.assertEqual(fast.tokenize("abcd gym"), ["abcd", "▁gym"]) + self.assertEqual( + fast.tokenize("Hi abcdgym world"), + ["▁Hi", "▁", "abcd", "gym", "▁world"], + ) + @require_tokenizers class TokenizerVersioningTest(unittest.TestCase): From 637e2cfe66331d6a2fcb86848f6d2042be2c42ce Mon Sep 17 00:00:00 2001 From: qflen Date: Tue, 12 May 2026 16:20:49 +0200 Subject: [PATCH 2/3] Simplify added-token prefix-space fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold the fix into `_get_prepend_scheme`: it now returns "first" instead of "always" when `add_prefix_space=True`, so standalone Metaspace pre_tokenizers minted via `convert_slow_tokenizer` no longer leave a spurious "▁" prefix on the chunk after an added token (#28218). "first" and "always" are identical on a standalone Metaspace for text without added tokens. WhitespaceSplit + Metaspace genuinely needs "always" so each split word keeps its "▁" SentencePiece marker; PegasusConverter and XLNetTokenizer now hardcode that scheme. Drop the `_init_complete` guard and `_align_metaspace_for_added_tokens` helper: the swap to "first" is inlined in `_add_tokens` and triggers when a non-special token is registered (covers .json loads that still bake in "always", whether the token comes from persisted state or a user `add_tokens` call). BigBird's persisted ``/`` slots are non-special, so the swap now runs at load time and the chunk after a literal `` loses its prefix; integration fixtures updated to match. --- src/transformers/convert_slow_tokenizer.py | 12 +++----- .../models/xlnet/tokenization_xlnet.py | 4 +-- src/transformers/tokenization_utils_base.py | 9 ++---- .../tokenization_utils_tokenizers.py | 29 +++++++------------ .../big_bird/test_tokenization_big_bird.py | 8 ++--- tests/tokenization/test_tokenization_fast.py | 5 ++-- 6 files changed, 25 insertions(+), 42 deletions(-) diff --git a/src/transformers/convert_slow_tokenizer.py b/src/transformers/convert_slow_tokenizer.py index 1d96d1c4d9a2..29c92adf5b85 100644 --- a/src/transformers/convert_slow_tokenizer.py +++ b/src/transformers/convert_slow_tokenizer.py @@ -110,13 +110,8 @@ def import_protobuf(error_message=""): def _get_prepend_scheme(add_prefix_space: bool, original_tokenizer) -> str: - if add_prefix_space: - prepend_scheme = "always" - if not getattr(original_tokenizer, "legacy", True): - prepend_scheme = "first" - else: - prepend_scheme = "never" - return prepend_scheme + # "first" avoids the spurious "▁" prefix on chunks following a user-added token (#28218). + return "first" if add_prefix_space else "never" def generate_merges(vocab, vocab_scores, skip_tokens: Collection[str] | None = None): @@ -1316,7 +1311,8 @@ def unk_id(self, proto): return proto.trainer_spec.unk_id + self.original_tokenizer.offset def pre_tokenizer(self, replacement, add_prefix_space): - prepend_scheme = _get_prepend_scheme(add_prefix_space, self.original_tokenizer) + # WhitespaceSplit + Metaspace needs "always" so each split word gets the "▁" marker. + prepend_scheme = "always" if add_prefix_space else "never" return pre_tokenizers.Sequence( [ pre_tokenizers.WhitespaceSplit(), diff --git a/src/transformers/models/xlnet/tokenization_xlnet.py b/src/transformers/models/xlnet/tokenization_xlnet.py index 0b682564bffd..dee73617198d 100644 --- a/src/transformers/models/xlnet/tokenization_xlnet.py +++ b/src/transformers/models/xlnet/tokenization_xlnet.py @@ -16,7 +16,6 @@ from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, processors from tokenizers.models import Unigram -from ...tokenization_utils_base import _get_prepend_scheme from ...tokenization_utils_tokenizers import TokenizersBackend from ...utils import logging @@ -144,7 +143,8 @@ def __init__( self._tokenizer.normalizer = normalizers.Sequence(list_normalizers) add_prefix_space = True - prepend_scheme = _get_prepend_scheme(add_prefix_space, self) + # WhitespaceSplit + Metaspace needs "always" so each split word gets the "▁" marker. + prepend_scheme = "always" if add_prefix_space else "never" self._tokenizer.pre_tokenizer = pre_tokenizers.Sequence( [ pre_tokenizers.WhitespaceSplit(), diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py index 4e821dfd4e70..02062ce9828c 100644 --- a/src/transformers/tokenization_utils_base.py +++ b/src/transformers/tokenization_utils_base.py @@ -3547,13 +3547,8 @@ def load_vocab_and_merges(pretrained_model_name_or_path, **kwargs): def _get_prepend_scheme(add_prefix_space: bool, original_tokenizer) -> str: - if add_prefix_space: - prepend_scheme = "always" - if not getattr(original_tokenizer, "legacy", True): - prepend_scheme = "first" - else: - prepend_scheme = "never" - return prepend_scheme + # "first" avoids the spurious "▁" prefix on chunks following a user-added token (#28218). + return "first" if add_prefix_space else "never" def generate_merges(vocab, vocab_scores: dict[str, float] | None = None, skip_tokens: Collection[str] | None = None): diff --git a/src/transformers/tokenization_utils_tokenizers.py b/src/transformers/tokenization_utils_tokenizers.py index 43d55e3472db..19ca2276811e 100644 --- a/src/transformers/tokenization_utils_tokenizers.py +++ b/src/transformers/tokenization_utils_tokenizers.py @@ -488,8 +488,6 @@ def __init__(self, *args, **kwargs): if self._should_update_post_processor: self.update_post_processor() - self._init_complete = True - @property def is_fast(self) -> bool: return True @@ -729,24 +727,19 @@ def _add_tokens(self, new_tokens: list[str | AddedToken], special_tokens=False) if special_tokens: return self._tokenizer.add_special_tokens(new_tokens) - if getattr(self, "_init_complete", False) and any(isinstance(t, str) or not t.special for t in new_tokens): - self._align_metaspace_for_added_tokens() + # For tokenizers loaded from a .json that bakes in prepend_scheme="always", + # swap a standalone Metaspace to "first" once a non-special token enters so + # the chunk after the added token does not pick up a spurious "▁" (#28218). + if any(isinstance(t, str) or not t.special for t in new_tokens): + pre_tokenizer = self._tokenizer.pre_tokenizer + if isinstance(pre_tokenizer, pre_tokenizers_fast.Metaspace) and pre_tokenizer.prepend_scheme == "always": + self._tokenizer.pre_tokenizer = pre_tokenizers_fast.Metaspace( + replacement=pre_tokenizer.replacement, + prepend_scheme="first", + split=pre_tokenizer.split, + ) return self._tokenizer.add_tokens(new_tokens) - def _align_metaspace_for_added_tokens(self) -> None: - # Mirror `SpmConverter.pre_tokenizer` for non-legacy spm tokenizers: switch a - # standalone Metaspace pre_tokenizer from prepend_scheme="always" to "first" - # so chunks following a user-added token are not given a spurious "▁" prefix. - # Sequence pipelines (eg. WhitespaceSplit + Metaspace) rely on "always" - # semantics for whitespace handling and are intentionally skipped. - pre_tokenizer = self._tokenizer.pre_tokenizer - if isinstance(pre_tokenizer, pre_tokenizers_fast.Metaspace) and pre_tokenizer.prepend_scheme == "always": - self._tokenizer.pre_tokenizer = pre_tokenizers_fast.Metaspace( - replacement=pre_tokenizer.replacement, - prepend_scheme="first", - split=pre_tokenizer.split, - ) - def num_special_tokens_to_add(self, pair: bool = False) -> int: """ Returns the number of added tokens when encoding a sequence with special tokens. diff --git a/tests/models/big_bird/test_tokenization_big_bird.py b/tests/models/big_bird/test_tokenization_big_bird.py index a12dc18fb033..b78f04533f96 100644 --- a/tests/models/big_bird/test_tokenization_big_bird.py +++ b/tests/models/big_bird/test_tokenization_big_bird.py @@ -31,7 +31,7 @@ class BigBirdTokenizationTest(TokenizerTesterMixin, unittest.TestCase): from_pretrained_id = "google/bigbird-roberta-base" tokenizer_class = BigBirdTokenizer - integration_expected_tokens = ['▁This', '▁is', '▁a', '▁test', '▁', '😊\n', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fals', 'é', '.', '\n生活的真谛是\n', 'Hi', '▁Hello', '\n', 'Hi', '▁Hello', '\n\n', '▁', '\n', '▁', '\n', '▁Hello', '\n', '', '▁', '\n', 'hi', '', '▁there', '\n', 'The', '▁following', '▁string', '▁should', '▁be', '▁properly', '▁encoded', ':', '▁Hello', '.', '\n', 'But', '▁', 'ird', '▁and', '▁', 'ปี', '▁', 'ird', '▁', 'ด\n', 'Hey', '▁how', '▁are', '▁you', '▁doing'] # fmt: skip - integration_expected_token_ids = [871, 419, 358, 1433, 321, 100, 141, 474, 4743, 388, 961, 11125, 112, 391, 529, 419, 27908, 266, 114, 100, 17351, 18536, 100, 17351, 18536, 100, 321, 100, 321, 100, 18536, 100, 2, 321, 100, 5404, 2, 713, 100, 565, 1809, 4832, 916, 408, 6206, 30341, 126, 18536, 114, 100, 1638, 321, 1548, 391, 321, 100, 321, 1548, 321, 100, 10915, 804, 490, 446, 1905] # fmt: skip - expected_tokens_from_ids = ['▁This', '▁is', '▁a', '▁test', '▁', '', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fals', 'é', '.', '', 'Hi', '▁Hello', '', 'Hi', '▁Hello', '', '▁', '', '▁', '', '▁Hello', '', '', '▁', '', 'hi', '', '▁there', '', 'The', '▁following', '▁string', '▁should', '▁be', '▁properly', '▁encoded', ':', '▁Hello', '.', '', 'But', '▁', 'ird', '▁and', '▁', '', '▁', 'ird', '▁', '', 'Hey', '▁how', '▁are', '▁you', '▁doing'] # fmt: skip - integration_expected_decoded_text = "This is a test I was born in 92000, and this is falsé.Hi HelloHi Hello Hello hi thereThe following string should be properly encoded: Hello.But ird and ird Hey how are you doing" + integration_expected_tokens = ['▁This', '▁is', '▁a', '▁test', '▁', '😊\n', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fals', 'é', '.', '\n生活的真谛是\n', 'Hi', '▁Hello', '\n', 'Hi', '▁Hello', '\n\n', '▁', '\n', '▁', '\n', '▁Hello', '\n', '', '\n', 'hi', '', 'there', '\n', 'The', '▁following', '▁string', '▁should', '▁be', '▁properly', '▁encoded', ':', '▁Hello', '.', '\n', 'But', '▁', 'ird', '▁and', '▁', 'ปี', '▁', 'ird', '▁', 'ด\n', 'Hey', '▁how', '▁are', '▁you', '▁doing'] # fmt: skip + integration_expected_token_ids = [871, 419, 358, 1433, 321, 100, 141, 474, 4743, 388, 961, 11125, 112, 391, 529, 419, 27908, 266, 114, 100, 17351, 18536, 100, 17351, 18536, 100, 321, 100, 321, 100, 18536, 100, 2, 100, 5404, 2, 8218, 100, 565, 1809, 4832, 916, 408, 6206, 30341, 126, 18536, 114, 100, 1638, 321, 1548, 391, 321, 100, 321, 1548, 321, 100, 10915, 804, 490, 446, 1905] # fmt: skip + expected_tokens_from_ids = ['▁This', '▁is', '▁a', '▁test', '▁', '', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fals', 'é', '.', '', 'Hi', '▁Hello', '', 'Hi', '▁Hello', '', '▁', '', '▁', '', '▁Hello', '', '', '', 'hi', '', 'there', '', 'The', '▁following', '▁string', '▁should', '▁be', '▁properly', '▁encoded', ':', '▁Hello', '.', '', 'But', '▁', 'ird', '▁and', '▁', '', '▁', 'ird', '▁', '', 'Hey', '▁how', '▁are', '▁you', '▁doing'] # fmt: skip + integration_expected_decoded_text = "This is a test I was born in 92000, and this is falsé.Hi HelloHi Hello HellohithereThe following string should be properly encoded: Hello.But ird and ird Hey how are you doing" diff --git a/tests/tokenization/test_tokenization_fast.py b/tests/tokenization/test_tokenization_fast.py index 8489e7de53a1..f68a8a9cad35 100644 --- a/tests/tokenization/test_tokenization_fast.py +++ b/tests/tokenization/test_tokenization_fast.py @@ -218,9 +218,8 @@ def test_class_after_save_and_reload(self): self.assertIsInstance(tokenizer, PreTrainedTokenizerFast) def test_added_token_does_not_get_metaspace_prefix(self): - # Issue #28218: when a non-special token is added to a SentencePiece-style fast - # tokenizer, the chunk that follows the added token must not be given a - # spurious "▁" prefix from the Metaspace pre_tokenizer. + # #28218: a non-special add_tokens call on a SentencePiece-style fast tokenizer + # must not leave a spurious "▁" prefix on the chunk after the added token. vocab = [ ("", 0.0), ("▁", -10.0), From d9f40a470775e8c24cb38f5134ab39677f6a65bc Mon Sep 17 00:00:00 2001 From: qflen Date: Sat, 4 Jul 2026 19:16:53 +0200 Subject: [PATCH 3/3] Realign prepend_scheme only for genuinely new post-init tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `_add_tokens` override flipped a standalone Metaspace("always") to "first" on any non-special add, including checkpoint restore. A tokenizer that loads as a standalone Metaspace (e.g. kosmos-2) re-registers its own added tokens both inside and after `__init__`, so the flip rewrote the saved scheme and dropped the "▁" after in-text special tokens, breaking Kosmos2ProcessorTest::test_full_processor. Now only realign when a genuinely new (not-yet-present) non-special token is added to a finished tokenizer: restoring a checkpoint's own vocab is left alone, while a real user `add_tokens` still gets the #28218 fix. Add a checkpoint-restore regression test. Also restore big_bird's integration expectations. main refactored it into a self-contained tokenizer that hardcodes prepend_scheme="always", so it no longer takes the converter's "first" path; it still gets the fix via `_add_tokens` on a real user add. --- .../tokenization_utils_tokenizers.py | 21 ++++++++++++++----- .../big_bird/test_tokenization_big_bird.py | 8 +++---- tests/tokenization/test_tokenization_fast.py | 21 +++++++++++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/transformers/tokenization_utils_tokenizers.py b/src/transformers/tokenization_utils_tokenizers.py index b11f2e01432f..68c58567cd83 100644 --- a/src/transformers/tokenization_utils_tokenizers.py +++ b/src/transformers/tokenization_utils_tokenizers.py @@ -488,6 +488,10 @@ def __init__(self, *args, **kwargs): if self._should_update_post_processor: self.update_post_processor() + # Construction (incl. checkpoint-restore of added tokens) is done; later non-special adds are + # genuine user edits and may realign the Metaspace prepend_scheme in `_add_tokens` (#28218). + self._prepend_scheme_can_flip = True + @property def is_fast(self) -> bool: return True @@ -727,12 +731,19 @@ def _add_tokens(self, new_tokens: list[str | AddedToken], special_tokens=False) if special_tokens: return self._tokenizer.add_special_tokens(new_tokens) - # For tokenizers loaded from a .json that bakes in prepend_scheme="always", - # swap a standalone Metaspace to "first" once a non-special token enters so - # the chunk after the added token does not pick up a spurious "▁" (#28218). - if any(isinstance(t, str) or not t.special for t in new_tokens): + # #28218: adding a genuinely new non-special token to a standalone Metaspace("always") leaves a + # spurious "▁" on the next chunk; realign to "first". Gated on a finished tokenizer and a not-yet- + # present token, so restoring a checkpoint's own added tokens (during or after init) is left alone. + if getattr(self, "_prepend_scheme_can_flip", False): pre_tokenizer = self._tokenizer.pre_tokenizer - if isinstance(pre_tokenizer, pre_tokenizers_fast.Metaspace) and pre_tokenizer.prepend_scheme == "always": + if ( + isinstance(pre_tokenizer, pre_tokenizers_fast.Metaspace) + and pre_tokenizer.prepend_scheme == "always" + and any( + (isinstance(t, str) or not t.special) and self._tokenizer.token_to_id(str(t)) is None + for t in new_tokens + ) + ): self._tokenizer.pre_tokenizer = pre_tokenizers_fast.Metaspace( replacement=pre_tokenizer.replacement, prepend_scheme="first", diff --git a/tests/models/big_bird/test_tokenization_big_bird.py b/tests/models/big_bird/test_tokenization_big_bird.py index b78f04533f96..a12dc18fb033 100644 --- a/tests/models/big_bird/test_tokenization_big_bird.py +++ b/tests/models/big_bird/test_tokenization_big_bird.py @@ -31,7 +31,7 @@ class BigBirdTokenizationTest(TokenizerTesterMixin, unittest.TestCase): from_pretrained_id = "google/bigbird-roberta-base" tokenizer_class = BigBirdTokenizer - integration_expected_tokens = ['▁This', '▁is', '▁a', '▁test', '▁', '😊\n', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fals', 'é', '.', '\n生活的真谛是\n', 'Hi', '▁Hello', '\n', 'Hi', '▁Hello', '\n\n', '▁', '\n', '▁', '\n', '▁Hello', '\n', '', '\n', 'hi', '', 'there', '\n', 'The', '▁following', '▁string', '▁should', '▁be', '▁properly', '▁encoded', ':', '▁Hello', '.', '\n', 'But', '▁', 'ird', '▁and', '▁', 'ปี', '▁', 'ird', '▁', 'ด\n', 'Hey', '▁how', '▁are', '▁you', '▁doing'] # fmt: skip - integration_expected_token_ids = [871, 419, 358, 1433, 321, 100, 141, 474, 4743, 388, 961, 11125, 112, 391, 529, 419, 27908, 266, 114, 100, 17351, 18536, 100, 17351, 18536, 100, 321, 100, 321, 100, 18536, 100, 2, 100, 5404, 2, 8218, 100, 565, 1809, 4832, 916, 408, 6206, 30341, 126, 18536, 114, 100, 1638, 321, 1548, 391, 321, 100, 321, 1548, 321, 100, 10915, 804, 490, 446, 1905] # fmt: skip - expected_tokens_from_ids = ['▁This', '▁is', '▁a', '▁test', '▁', '', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fals', 'é', '.', '', 'Hi', '▁Hello', '', 'Hi', '▁Hello', '', '▁', '', '▁', '', '▁Hello', '', '', '', 'hi', '', 'there', '', 'The', '▁following', '▁string', '▁should', '▁be', '▁properly', '▁encoded', ':', '▁Hello', '.', '', 'But', '▁', 'ird', '▁and', '▁', '', '▁', 'ird', '▁', '', 'Hey', '▁how', '▁are', '▁you', '▁doing'] # fmt: skip - integration_expected_decoded_text = "This is a test I was born in 92000, and this is falsé.Hi HelloHi Hello HellohithereThe following string should be properly encoded: Hello.But ird and ird Hey how are you doing" + integration_expected_tokens = ['▁This', '▁is', '▁a', '▁test', '▁', '😊\n', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fals', 'é', '.', '\n生活的真谛是\n', 'Hi', '▁Hello', '\n', 'Hi', '▁Hello', '\n\n', '▁', '\n', '▁', '\n', '▁Hello', '\n', '', '▁', '\n', 'hi', '', '▁there', '\n', 'The', '▁following', '▁string', '▁should', '▁be', '▁properly', '▁encoded', ':', '▁Hello', '.', '\n', 'But', '▁', 'ird', '▁and', '▁', 'ปี', '▁', 'ird', '▁', 'ด\n', 'Hey', '▁how', '▁are', '▁you', '▁doing'] # fmt: skip + integration_expected_token_ids = [871, 419, 358, 1433, 321, 100, 141, 474, 4743, 388, 961, 11125, 112, 391, 529, 419, 27908, 266, 114, 100, 17351, 18536, 100, 17351, 18536, 100, 321, 100, 321, 100, 18536, 100, 2, 321, 100, 5404, 2, 713, 100, 565, 1809, 4832, 916, 408, 6206, 30341, 126, 18536, 114, 100, 1638, 321, 1548, 391, 321, 100, 321, 1548, 321, 100, 10915, 804, 490, 446, 1905] # fmt: skip + expected_tokens_from_ids = ['▁This', '▁is', '▁a', '▁test', '▁', '', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fals', 'é', '.', '', 'Hi', '▁Hello', '', 'Hi', '▁Hello', '', '▁', '', '▁', '', '▁Hello', '', '', '▁', '', 'hi', '', '▁there', '', 'The', '▁following', '▁string', '▁should', '▁be', '▁properly', '▁encoded', ':', '▁Hello', '.', '', 'But', '▁', 'ird', '▁and', '▁', '', '▁', 'ird', '▁', '', 'Hey', '▁how', '▁are', '▁you', '▁doing'] # fmt: skip + integration_expected_decoded_text = "This is a test I was born in 92000, and this is falsé.Hi HelloHi Hello Hello hi thereThe following string should be properly encoded: Hello.But ird and ird Hey how are you doing" diff --git a/tests/tokenization/test_tokenization_fast.py b/tests/tokenization/test_tokenization_fast.py index f68a8a9cad35..c4121d183561 100644 --- a/tests/tokenization/test_tokenization_fast.py +++ b/tests/tokenization/test_tokenization_fast.py @@ -243,6 +243,27 @@ def test_added_token_does_not_get_metaspace_prefix(self): ["▁Hi", "▁", "abcd", "gym", "▁world"], ) + def test_checkpoint_restore_keeps_metaspace_prepend_scheme(self): + # #28218 follow-up: restoring a checkpoint that legitimately baked prepend_scheme="always" + # together with its own non-special added tokens must NOT flip the scheme to "first". The + # realignment only applies to genuine post-init user add_tokens, never to load-time restore. + vocab = [("", 0.0), ("▁", -10.0), ("gym", 0.0), ("▁gym", 0.0)] + backend = Tokenizer(Unigram(vocab=vocab, unk_id=0)) + backend.pre_tokenizer = pre_tokenizers.Metaspace(replacement="▁", prepend_scheme="always", split=True) + backend.add_tokens(["abcd"]) # a non-special token baked into the checkpoint + fast = PreTrainedTokenizerFast(tokenizer_object=backend, unk_token="") + self.assertEqual(fast._tokenizer.pre_tokenizer.prepend_scheme, "always") + + with tempfile.TemporaryDirectory() as tmp_dir: + fast.save_pretrained(tmp_dir) + reloaded = PreTrainedTokenizerFast.from_pretrained(tmp_dir) + + # Restoring "abcd" while loading must leave the scheme untouched... + self.assertEqual(reloaded._tokenizer.pre_tokenizer.prepend_scheme, "always") + # ...while a genuine post-load user add still realigns it. + reloaded.add_tokens(["efgh"]) + self.assertEqual(reloaded._tokenizer.pre_tokenizer.prepend_scheme, "first") + def test_bpe_tokenizer_skips_clean_up_tokenization_spaces(self): """BPE tokenizers should not apply clean_up_tokenization even when the flag is True.