From aed9d75e2e6bf736228d61440fb1b09f90200a44 Mon Sep 17 00:00:00 2001 From: phoneee Date: Sun, 29 Mar 2026 18:09:29 +0700 Subject: [PATCH] fix: handle empty strings in word_detokenize to prevent IndexError fix: guard word_detokenize against empty list input --- pythainlp/tokenize/core.py | 7 ++++++- tests/core/test_tokenize.py | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index ff133c68b..845b3d7b2 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -52,6 +52,9 @@ def word_detokenize( """ list_all: list[list[str]] = [] + if not segments: + return "" if output == "str" else [] + if isinstance(segments[0], str): segments = [segments] # type: ignore[assignment] @@ -63,6 +66,8 @@ def word_detokenize( space_index: list[int] = [] mark_index: list[int] = [] for j, w in enumerate(s): + if not w: + continue if j > 0: # previous word p_w = s[j - 1] @@ -75,7 +80,7 @@ def word_detokenize( list_sents.append(" ") add_index.append(j) # if previous word is number or other language and is not space - elif p_w[0] not in thai_characters and not p_w.isspace(): + elif p_w and p_w[0] not in thai_characters and not p_w.isspace(): list_sents.append(" ") add_index.append(j) # if word is Thai iteration mark diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py index 0b5befd3a..2692c240c 100644 --- a/tests/core/test_tokenize.py +++ b/tests/core/test_tokenize.py @@ -233,6 +233,12 @@ def test_word_detokenize(self): word_detokenize(["ม่ายย", " ", "ผม", "เลี้ยง", "5", "ตัว"]), "ม่ายย ผมเลี้ยง 5 ตัว", ) + # Reproduce: empty strings in token list should not cause IndexError + self.assertIsInstance(word_detokenize(["สวัสดี", "", "ครับ"]), str) + self.assertIsInstance(word_detokenize(["hello", "", "world"]), str) + self.assertIsInstance(word_detokenize([""]), str) + # Empty list should not crash + self.assertEqual(word_detokenize([]), "") def test_numeric_data_format(self): engines = ["newmm", "longest"]