Merge pull request #1374 from phoneee/fix/word-detokenize-empty-string

fix: handle empty strings and empty list in word_detokenize

Author: bact

pythainlp/tokenize/core.py

@@ -52,6 +52,9 @@ def word_detokenize(
     """
     list_all: list[list[str]] = []
 
+    if not segments:
+        return "" if output == "str" else []
+
     if isinstance(segments[0], str):
         segments = [segments]  # type: ignore[assignment]
 
@@ -63,6 +66,8 @@ def word_detokenize(
         space_index: list[int] = []
         mark_index: list[int] = []
         for j, w in enumerate(s):
+            if not w:
+                continue
             if j > 0:
                 # previous word
                 p_w = s[j - 1]
@@ -75,7 +80,7 @@ def word_detokenize(
                     list_sents.append(" ")
                     add_index.append(j)
                 # if previous word is number or other language and is not space
-                elif p_w[0] not in thai_characters and not p_w.isspace():
+                elif p_w and p_w[0] not in thai_characters and not p_w.isspace():
                     list_sents.append(" ")
                     add_index.append(j)
                 # if word is Thai iteration mark

tests/core/test_tokenize.py

@@ -233,6 +233,12 @@ def test_word_detokenize(self):
             word_detokenize(["ม่ายย", " ", "ผม", "เลี้ยง", "5", "ตัว"]),
             "ม่ายย ผมเลี้ยง 5 ตัว",
         )
+        # Reproduce: empty strings in token list should not cause IndexError
+        self.assertIsInstance(word_detokenize(["สวัสดี", "", "ครับ"]), str)
+        self.assertIsInstance(word_detokenize(["hello", "", "world"]), str)
+        self.assertIsInstance(word_detokenize([""]), str)
+        # Empty list should not crash
+        self.assertEqual(word_detokenize([]), "")
 
     def test_numeric_data_format(self):
         engines = ["newmm", "longest"]