Merge pull request #1382 from phoneee/fix/tokenize-defensive-guards

fix: guard han_solo list_cut[-1] and narrow longest.py exception catch

Author: bact

pythainlp/tokenize/han_solo.py

@@ -160,9 +160,9 @@ def segment(text: str) -> list[str]:
     tagger = _get_tagger()
     x = _to_feature.featurize(text)["X"]
     y_pred = tagger.tag(x)
-    list_cut = []
+    list_cut: list[str] = []
     for j, k in zip(list(text), y_pred):
-        if k == "1":
+        if k == "1" or not list_cut:
             list_cut.append(j)
         else:
             list_cut[-1] += j

pythainlp/tokenize/longest.py

@@ -105,7 +105,7 @@ def __longest_matching(self, text: str, begin_pos: int) -> str:
                     return text[0 : len_word_valid + 1]
                 else:
                     return word_valid
-            except BaseException:
+            except IndexError:
                 return word_valid
         else:
             return ""