diff --git a/pyproject.toml b/pyproject.toml
index 8a079a303..9f97d670e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -411,7 +411,6 @@ module = [
"khamyo.*",
"khanaa.*",
"multiel.*",
- "nlpo3.*",
"nltk.*",
"numpy.*",
"onnxruntime.*",
diff --git a/pythainlp/tokenize/_utils.py b/pythainlp/tokenize/_utils.py
index 7ef51d074..c9aa0f523 100644
--- a/pythainlp/tokenize/_utils.py
+++ b/pythainlp/tokenize/_utils.py
@@ -7,13 +7,13 @@
from __future__ import annotations
import re
-from collections.abc import Callable
+from collections.abc import Callable, Sequence
_DIGITS_WITH_SEPARATOR = re.compile(r"(\d+[\.\,:])+\d+")
def apply_postprocessors(
- segments: list[str], postprocessors: list[Callable[[list[str]], list[str]]]
+ segments: list[str], postprocessors: Sequence[Callable[[list[str]], list[str]]]
) -> list[str]:
"""A list of callables to apply to a raw segmentation result.
"""
diff --git a/pythainlp/tokenize/attacut.py b/pythainlp/tokenize/attacut.py
index 6c31cc0bd..e215e1663 100644
--- a/pythainlp/tokenize/attacut.py
+++ b/pythainlp/tokenize/attacut.py
@@ -10,6 +10,7 @@
from __future__ import annotations
import threading
+from typing import cast
from attacut import Tokenizer
@@ -24,7 +25,7 @@ def __init__(self, model="attacut-sc"):
self._tokenizer = Tokenizer(model=self._MODEL_NAME)
def tokenize(self, text: str) -> list[str]:
- return self._tokenizer.tokenize(text)
+ return cast(list[str], self._tokenizer.tokenize(text))
_tokenizers: dict[str, AttacutTokenizer] = {}
diff --git a/pythainlp/tokenize/budoux.py b/pythainlp/tokenize/budoux.py
index 85b02cd68..2c6b95e1e 100644
--- a/pythainlp/tokenize/budoux.py
+++ b/pythainlp/tokenize/budoux.py
@@ -13,6 +13,7 @@
from __future__ import annotations
import threading
+from typing import cast
_parser = None
_parser_lock = threading.Lock()
@@ -55,6 +56,6 @@ def segment(text: str) -> list[str]:
_parser = _init_parser()
parser = _parser
- result = parser.parse(text)
+ result = cast(list[str], parser.parse(text))
return result
diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py
index bcb76aa72..5a7aa1cb9 100644
--- a/pythainlp/tokenize/core.py
+++ b/pythainlp/tokenize/core.py
@@ -32,7 +32,7 @@
def word_detokenize(
segments: Union[list[list[str]], list[str]], output: str = "str"
-) -> Union[list[str], str]:
+) -> Union[list[list[str]], str]:
"""Word detokenizer.
Detokenizes the list of words in each sentence into text.
@@ -49,18 +49,18 @@ def word_detokenize(
print(word_detokenize(["เรา", "เล่น"]))
# output: เราเล่น
"""
- list_all = []
+ list_all: list[list[str]] = []
if isinstance(segments[0], str):
- segments = [segments]
+ segments = [segments] # type: ignore[assignment]
from pythainlp import thai_characters
for i, s in enumerate(segments):
- list_sents = []
- add_index = []
- space_index = []
- mark_index = []
+ list_sents: list[str] = []
+ add_index: list[int] = []
+ space_index: list[int] = []
+ mark_index: list[int] = []
for j, w in enumerate(s):
if j > 0:
# previous word
@@ -92,9 +92,9 @@ def word_detokenize(
if output == "list":
return list_all
- text = []
- for i in list_all:
- text.append("".join(i))
+ text: list[str] = []
+ for sent_tokens in list_all:
+ text.append("".join(sent_tokens))
return " ".join(text)
@@ -259,56 +259,56 @@ def word_tokenize(
segments = segment(text, custom_dict, safe_mode=True)
elif engine == "attacut":
- from pythainlp.tokenize.attacut import segment
+ from pythainlp.tokenize.attacut import segment as attacut_segment # noqa: I001
- segments = segment(text)
+ segments = attacut_segment(text)
elif engine == "longest":
- from pythainlp.tokenize.longest import segment
+ from pythainlp.tokenize.longest import segment as longest_segment # noqa: I001
- segments = segment(text, custom_dict)
+ segments = longest_segment(text, custom_dict)
elif engine in ("mm", "multi_cut"):
- from pythainlp.tokenize.multi_cut import segment
+ from pythainlp.tokenize.multi_cut import segment as multi_cut_segment # noqa: I001
- segments = segment(text, custom_dict)
+ segments = multi_cut_segment(text, custom_dict)
elif engine == "deepcut": # deepcut can optionally use dictionary
- from pythainlp.tokenize.deepcut import segment
+ from pythainlp.tokenize.deepcut import segment as deepcut_segment # noqa: I001
if custom_dict:
- custom_dict = list(custom_dict)
- segments = segment(text, custom_dict)
+ custom_dict = list(custom_dict) # type: ignore[assignment]
+ segments = deepcut_segment(text, custom_dict)
else:
- segments = segment(text)
+ segments = deepcut_segment(text)
elif engine == "icu":
- from pythainlp.tokenize.pyicu import segment
+ from pythainlp.tokenize.pyicu import segment as pyicu_segment # noqa: I001
- segments = segment(text)
+ segments = pyicu_segment(text)
elif engine == "budoux":
- from pythainlp.tokenize.budoux import segment
+ from pythainlp.tokenize.budoux import segment as budoux_segment # noqa: I001
- segments = segment(text)
+ segments = budoux_segment(text)
elif engine == "nercut":
- from pythainlp.tokenize.nercut import segment
+ from pythainlp.tokenize.nercut import segment as nercut_segment # noqa: I001
- segments = segment(text)
+ segments = nercut_segment(text)
elif engine == "sefr_cut":
- from pythainlp.tokenize.sefr_cut import segment
+ from pythainlp.tokenize.sefr_cut import segment as sefrcut_segment # noqa: I001
- segments = segment(text)
+ segments = sefrcut_segment(text)
elif engine == "tltk":
- from pythainlp.tokenize.tltk import segment
+ from pythainlp.tokenize.tltk import segment as tltk_segment # noqa: I001
- segments = segment(text)
+ segments = tltk_segment(text)
elif engine == "oskut":
- from pythainlp.tokenize.oskut import segment
+ from pythainlp.tokenize.oskut import segment as oskut_segment # noqa: I001
- segments = segment(text)
+ segments = oskut_segment(text)
elif engine == "nlpo3":
- from pythainlp.tokenize.nlpo3 import segment
+ from pythainlp.tokenize.nlpo3 import segment as nlpo3_segment # noqa: I001
# Currently cannot handle custom_dict from inside word_tokenize(),
# due to difference in type.
# if isinstance(custom_dict, str):
- # segments = segment(text, custom_dict=custom_dict)
+ # segments = nlpo3_segment(text, custom_dict=custom_dict)
# elif not isinstance(custom_dict, str) and not custom_dict:
# raise ValueError(
# f"""Tokenizer \"{engine}\":
@@ -317,8 +317,8 @@ def word_tokenize(
# See pythainlp.tokenize.nlpo3.load_dict()"""
# )
# else:
- # segments = segment(text)
- segments = segment(text)
+ # segments = nlpo3_segment(text)
+ segments = nlpo3_segment(text)
else:
raise ValueError(
f"""Tokenizer \"{engine}\" not found.
@@ -413,7 +413,7 @@ def sent_tokenize(
text: Union[str, list[str]],
engine: str = DEFAULT_SENT_TOKENIZE_ENGINE,
keep_whitespace: bool = True,
-) -> list[str]:
+) -> Union[list[str], list[list[str]]]:
"""Sentence tokenizer.
Tokenizes running text into "sentences". Supports both string and list of strings.
@@ -632,7 +632,7 @@ def paragraph_tokenize(
It might be a typo; if not, please consult our document."""
)
- return segments
+ return segments # type: ignore[return-value]
def subword_tokenize(
@@ -719,13 +719,17 @@ def subword_tokenize(
segments = []
if engine == "tcc":
- from pythainlp.tokenize.tcc import segment
+ from pythainlp.tokenize.tcc import segment as tcc_segment
+ segments = tcc_segment(text)
elif engine == "tcc_p":
- from pythainlp.tokenize.tcc_p import segment
+ from pythainlp.tokenize.tcc_p import segment as tcc_p_segment
+ segments = tcc_p_segment(text)
elif engine == "etcc":
- from pythainlp.tokenize.etcc import segment
+ from pythainlp.tokenize.etcc import segment as etcc_segment
+ segments = etcc_segment(text)
elif engine == "wangchanberta":
- from pythainlp.wangchanberta import segment
+ from pythainlp.wangchanberta import segment as wangchanberta_segment
+ segments = wangchanberta_segment(text)
elif engine == "dict": # use syllable dictionary
words = word_tokenize(text)
for word in words:
@@ -733,22 +737,23 @@ def subword_tokenize(
word_tokenize(text=word, custom_dict=syllable_dict_trie())
)
elif engine == "ssg":
- from pythainlp.tokenize.ssg import segment
+ from pythainlp.tokenize.ssg import segment as ssg_segment
+ segments = ssg_segment(text)
elif engine == "tltk":
- from pythainlp.tokenize.tltk import syllable_tokenize as segment
+ from pythainlp.tokenize.tltk import syllable_tokenize as tltk_segment
+ segments = tltk_segment(text)
elif engine == "han_solo":
- from pythainlp.tokenize.han_solo import segment
+ from pythainlp.tokenize.han_solo import segment as han_solo_segment
+ segments = han_solo_segment(text)
elif engine == "phayathai":
- from pythainlp.phayathaibert import segment
+ from pythainlp.phayathaibert import segment as phayathai_segment
+ segments = phayathai_segment(text)
else:
raise ValueError(
f"""Tokenizer \"{engine}\" not found.
It might be a typo; if not, please consult our document."""
)
- if not segments:
- segments = segment(text)
-
if not keep_whitespace:
segments = strip_whitespace(segments)
diff --git a/pythainlp/tokenize/crfcut.py b/pythainlp/tokenize/crfcut.py
index bbf437b05..1150bb0a9 100644
--- a/pythainlp/tokenize/crfcut.py
+++ b/pythainlp/tokenize/crfcut.py
@@ -183,10 +183,7 @@ def segment(text: str) -> list[str]:
:param str text: text to be tokenized into sentences
:return: list of words, tokenized from the text
"""
- if isinstance(text, str):
- toks = word_tokenize(text)
- else:
- toks = text
+ toks = word_tokenize(text)
feat = extract_features(toks)
labs = _tagger.tag(feat)
labs[-1] = "E" # make sure it cuts the last sentence
diff --git a/pythainlp/tokenize/deepcut.py b/pythainlp/tokenize/deepcut.py
index 0809c4b13..30a491670 100644
--- a/pythainlp/tokenize/deepcut.py
+++ b/pythainlp/tokenize/deepcut.py
@@ -12,7 +12,7 @@
from __future__ import annotations
-from typing import Union
+from typing import Union, cast
try:
from deepcut import tokenize
@@ -29,6 +29,6 @@ def segment(text: str, custom_dict: Union[Trie, list[str], str] = []) -> list[st
if isinstance(custom_dict, Trie):
custom_dict = list(custom_dict)
- return tokenize(text, custom_dict)
+ return cast(list[str], tokenize(text, custom_dict))
- return tokenize(text)
+ return cast(list[str], tokenize(text))
diff --git a/pythainlp/tokenize/longest.py b/pythainlp/tokenize/longest.py
index 90ca3cd0c..1712559be 100644
--- a/pythainlp/tokenize/longest.py
+++ b/pythainlp/tokenize/longest.py
@@ -106,7 +106,7 @@ def __longest_matching(self, text: str, begin_pos: int) -> str:
else:
return ""
- def __segment(self, text: str):
+ def __segment(self, text: str) -> list[str]:
begin_pos = 0
len_text = len(text)
tokens: list[str] = []
diff --git a/pythainlp/tokenize/nercut.py b/pythainlp/tokenize/nercut.py
index 50f33d8ba..0cf72d8a5 100644
--- a/pythainlp/tokenize/nercut.py
+++ b/pythainlp/tokenize/nercut.py
@@ -40,7 +40,7 @@ def segment(
:param class tagger: NER tagger engine
:return: list of words, tokenized from the text
"""
- if not isinstance(text, str):
+ if not text:
return []
tagged_words = tagger.tag(text, pos=False)
@@ -73,7 +73,5 @@ def segment(
words.append(combining_word)
elif curr_tag.startswith("I-") and combining_word != "":
words.append(combining_word)
- else:
- pass
return words
diff --git a/pythainlp/tokenize/nlpo3.py b/pythainlp/tokenize/nlpo3.py
index 55fa7c877..12ff94b82 100644
--- a/pythainlp/tokenize/nlpo3.py
+++ b/pythainlp/tokenize/nlpo3.py
@@ -6,9 +6,11 @@
import threading
from importlib.resources import as_file, files
from sys import stderr
+from typing import TYPE_CHECKING
-from nlpo3 import load_dict as nlpo3_load_dict
-from nlpo3 import segment as nlpo3_segment
+if TYPE_CHECKING:
+ from nlpo3 import load_dict as nlpo3_load_dict # noqa: F401
+ from nlpo3 import segment as nlpo3_segment # noqa: F401
from pythainlp.corpus.common import _THAI_WORDS_FILENAME
@@ -25,6 +27,13 @@ def _ensure_default_dict_loaded():
The context manager is kept alive for the lifetime of the program
to prevent cleanup of temporary files while the dictionary is in use.
"""
+ try:
+ from nlpo3 import load_dict as nlpo3_load_dict
+ except ImportError as ex:
+ raise ImportError(
+ "nlpo3 is not installed. Install it with: pip install nlpo3"
+ ) from ex
+
global _NLPO3_DEFAULT_DICT, _dict_file_ctx
if _NLPO3_DEFAULT_DICT is None:
with _load_lock:
@@ -57,6 +66,13 @@ def load_dict(file_path: str, dict_name: str) -> bool:
* \
https://github.com/PyThaiNLP/nlpo3
"""
+ try:
+ from nlpo3 import load_dict as nlpo3_load_dict
+ except ImportError as ex:
+ raise ImportError(
+ "nlpo3 is not installed. Install it with: pip install nlpo3"
+ ) from ex
+
msg, success = nlpo3_load_dict(file_path=file_path, dict_name=dict_name)
if not success:
print(msg, file=stderr)
@@ -87,6 +103,13 @@ def segment(
* \
https://github.com/PyThaiNLP/nlpo3
"""
+ try:
+ from nlpo3 import segment as nlpo3_segment
+ except ImportError as ex:
+ raise ImportError(
+ "nlpo3 is not installed. Install it with: pip install nlpo3"
+ ) from ex
+
# Ensure default dict is loaded if it's being used
if custom_dict == _NLPO3_DEFAULT_DICT_NAME:
_ensure_default_dict_loaded()
diff --git a/pythainlp/tokenize/oskut.py b/pythainlp/tokenize/oskut.py
index 55e48c617..5c5f47fa4 100644
--- a/pythainlp/tokenize/oskut.py
+++ b/pythainlp/tokenize/oskut.py
@@ -12,6 +12,7 @@
from __future__ import annotations
import threading
+from typing import cast
import oskut
@@ -45,4 +46,4 @@ def segment(text: str, engine: str = "ws") -> list[str]:
_DEFAULT_ENGINE = engine
oskut.load_model(engine=_DEFAULT_ENGINE)
- return oskut.OSKut(text)
+ return cast(list[str], oskut.OSKut(text))
diff --git a/pythainlp/tokenize/sefr_cut.py b/pythainlp/tokenize/sefr_cut.py
index a12cbf221..4803a3265 100644
--- a/pythainlp/tokenize/sefr_cut.py
+++ b/pythainlp/tokenize/sefr_cut.py
@@ -11,6 +11,7 @@
from __future__ import annotations
import threading
+from typing import cast
import sefr_cut
@@ -44,4 +45,4 @@ def segment(text: str, engine: str = "ws1000") -> list[str]:
_DEFAULT_ENGINE = engine
sefr_cut.load_model(engine=_DEFAULT_ENGINE)
- return sefr_cut.tokenize(text)[0]
+ return cast(list[str], sefr_cut.tokenize(text)[0])
diff --git a/pythainlp/tokenize/ssg.py b/pythainlp/tokenize/ssg.py
index ccaa56550..431a77032 100644
--- a/pythainlp/tokenize/ssg.py
+++ b/pythainlp/tokenize/ssg.py
@@ -3,6 +3,8 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
+from typing import cast
+
from ssg import syllable_tokenize
@@ -12,4 +14,4 @@ def segment(text: str) -> list[str]:
if not text or not isinstance(text, str):
return []
- return syllable_tokenize(text)
+ return cast(list[str], syllable_tokenize(text))
diff --git a/pythainlp/tokenize/tltk.py b/pythainlp/tokenize/tltk.py
index 90b1ab4b2..b345ee47a 100644
--- a/pythainlp/tokenize/tltk.py
+++ b/pythainlp/tokenize/tltk.py
@@ -16,27 +16,27 @@ def segment(text: str) -> list[str]:
if not text or not isinstance(text, str):
return []
text = text.replace(" ", "")
- _temp = tltk_segment(text).replace("", " ").replace("", "")
- _temp = _temp.split("|")
- if _temp[-1] == "":
- del _temp[-1]
- return _temp
+ _temp: str = tltk_segment(text).replace("", " ").replace("", "")
+ _temp_list = _temp.split("|")
+ if _temp_list[-1] == "":
+ del _temp_list[-1]
+ return _temp_list
def syllable_tokenize(text: str) -> list[str]:
if not text or not isinstance(text, str):
return []
- _temp = syl_segment(text)
- _temp = _temp.split("~")
- if _temp[-1] == "":
- del _temp[-1]
- return _temp
+ _temp: str = syl_segment(text)
+ _temp_list = _temp.split("~")
+ if _temp_list[-1] == "":
+ del _temp_list[-1]
+ return _temp_list
def sent_tokenize(text: str) -> list[str]:
text = text.replace(" ", "")
- _temp = tltk_segment(text).replace("", " ").replace("|", "")
- _temp = _temp.split("")
- if _temp[-1] == "":
- del _temp[-1]
- return _temp
+ _temp: str = tltk_segment(text).replace("", " ").replace("|", "")
+ _temp_list = _temp.split("")
+ if _temp_list[-1] == "":
+ del _temp_list[-1]
+ return _temp_list
diff --git a/pythainlp/tokenize/wtsplit.py b/pythainlp/tokenize/wtsplit.py
index 6d69c81c5..56791d443 100644
--- a/pythainlp/tokenize/wtsplit.py
+++ b/pythainlp/tokenize/wtsplit.py
@@ -9,6 +9,7 @@
from __future__ import annotations
import threading
+from typing import cast
from wtpsplit import WtP
@@ -45,22 +46,28 @@ def _tokenize(
raise RuntimeError("Model failed to load")
if tokenize == "sentence":
- return model_instance.split(text, lang_code=lang_code)
+ return cast(list[str], model_instance.split(text, lang_code=lang_code))
else: # Paragraph
if style == "newline":
- return model_instance.split(
- text,
- lang_code=lang_code,
- do_paragraph_segmentation=True,
- paragraph_threshold=paragraph_threshold,
+ return cast(
+ list[str],
+ model_instance.split(
+ text,
+ lang_code=lang_code,
+ do_paragraph_segmentation=True,
+ paragraph_threshold=paragraph_threshold,
+ ),
)
elif style == "opus100":
- return model_instance.split(
- text,
- lang_code=lang_code,
- do_paragraph_segmentation=True,
- threshold=paragraph_threshold,
- style=style,
+ return cast(
+ list[str],
+ model_instance.split(
+ text,
+ lang_code=lang_code,
+ do_paragraph_segmentation=True,
+ threshold=paragraph_threshold,
+ style=style,
+ ),
)
else:
raise ValueError(