diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index ff133c68b..845b3d7b2 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -52,6 +52,9 @@ def word_detokenize( """ list_all: list[list[str]] = [] + if not segments: + return "" if output == "str" else [] + if isinstance(segments[0], str): segments = [segments] # type: ignore[assignment] @@ -63,6 +66,8 @@ def word_detokenize( space_index: list[int] = [] mark_index: list[int] = [] for j, w in enumerate(s): + if not w: + continue if j > 0: # previous word p_w = s[j - 1] @@ -75,7 +80,7 @@ def word_detokenize( list_sents.append(" ") add_index.append(j) # if previous word is number or other language and is not space - elif p_w[0] not in thai_characters and not p_w.isspace(): + elif p_w and p_w[0] not in thai_characters and not p_w.isspace(): list_sents.append(" ") add_index.append(j) # if word is Thai iteration mark diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py index 0b5befd3a..2692c240c 100644 --- a/tests/core/test_tokenize.py +++ b/tests/core/test_tokenize.py @@ -233,6 +233,12 @@ def test_word_detokenize(self): word_detokenize(["ม่ายย", " ", "ผม", "เลี้ยง", "5", "ตัว"]), "ม่ายย ผมเลี้ยง 5 ตัว", ) + # Reproduce: empty strings in token list should not cause IndexError + self.assertIsInstance(word_detokenize(["สวัสดี", "", "ครับ"]), str) + self.assertIsInstance(word_detokenize(["hello", "", "world"]), str) + self.assertIsInstance(word_detokenize([""]), str) + # Empty list should not crash + self.assertEqual(word_detokenize([]), "") def test_numeric_data_format(self): engines = ["newmm", "longest"]