do not index past decoded chars with special tokens (huggingface#45435)

* do not index past decoded chars with special tokens

* better test explanation

Co-authored-by: Krishnachaitanyakc <22275437+Krishnachaitanyakc@users.noreply.github.com>

---------

Co-authored-by: Krishnachaitanyakc <22275437+Krishnachaitanyakc@users.noreply.github.com>

Author: itazap

src/transformers/models/whisper/tokenization_whisper.py

@@ -1331,6 +1331,7 @@ def _split_tokens_on_unicode(tokenizer, tokens: list[int]):
 
         if (
             replacement_char not in decoded
+            or unicode_offset + decoded.index(replacement_char) >= len(decoded_full)
             or decoded_full[unicode_offset + decoded.index(replacement_char)] == replacement_char
         ):
             words.append(decoded)

tests/models/whisper/test_tokenization_whisper.py

@@ -17,7 +17,11 @@
 import numpy as np
 
 from transformers.models.whisper import WhisperTokenizer
-from transformers.models.whisper.tokenization_whisper import _combine_tokens_into_words, _find_longest_common_sequence
+from transformers.models.whisper.tokenization_whisper import (
+    _combine_tokens_into_words,
+    _find_longest_common_sequence,
+    _split_tokens_on_unicode,
+)
 from transformers.testing_utils import require_torch, slow
 
 from ...test_tokenization_common import TokenizerTesterMixin
@@ -269,6 +273,30 @@ def test_combine_tokens_into_words(self):
         self.assertEqual(expected_tokens, output[1])
         self.assertEqual(expected_indices, output[2])
 
+    def test_split_tokens_on_unicode_trailing_replacement_char(self):
+        """Test `_split_tokens_on_unicode` with a trailing token that decodes to U+FFFD (Unicode replacement char)."""
+        from collections import defaultdict
+
+        class DummyTokenizer:
+            def __init__(self):
+                self.responses = defaultdict(list)
+
+            def decode(self, tokens, decode_with_timestamps=False):
+                key = tuple(tokens)
+                if self.responses[key]:
+                    return self.responses[key].pop(0)
+                return ""
+
+        tokenizer = DummyTokenizer()
+        tokenizer.responses[(1, 2)] = ["ab"]
+        tokenizer.responses[(1,)] = ["ab"]
+        tokenizer.responses[(2,)] = ["\ufffd"]
+
+        words, word_tokens, token_indices = _split_tokens_on_unicode(tokenizer, [1, 2])
+        self.assertEqual(words, ["ab", "\ufffd"])
+        self.assertEqual(word_tokens, [[1], [2]])
+        self.assertEqual(token_indices, [[0], [1]])
+
     def test_basic_normalizer(self):
         tokenizer = self.get_tokenizer()