From 8c1ee98ae2ec69b6ab63ac7e2695cac502a1ce4d Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 13:49:16 +0700 Subject: [PATCH 01/18] refactor: lazy load ipa cut --- pythainlp/util/phoneme.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pythainlp/util/phoneme.py b/pythainlp/util/phoneme.py index c473b2cd2..ecdb2d84f 100644 --- a/pythainlp/util/phoneme.py +++ b/pythainlp/util/phoneme.py @@ -5,6 +5,7 @@ """ Phonemes util """ +from functools import lru_cache import unicodedata from pythainlp.tokenize import Tokenizer @@ -192,8 +193,12 @@ def nectec_to_ipa(pronunciation: str) -> str: } dict_ipa_rtgs_final = {"w": "o"} -trie = Trie(list(dict_ipa_rtgs.keys()) + list(dict_ipa_rtgs_final.keys())) -ipa_cut = Tokenizer(custom_dict=trie, engine="newmm") + +@lru_cache +def _ipa_cut(): + """Lazy load IPA tokenizer with cache""" + trie = Trie(list(dict_ipa_rtgs.keys()) + list(dict_ipa_rtgs_final.keys())) + return Tokenizer(custom_dict=trie, engine="newmm") def ipa_to_rtgs(ipa: str) -> str: @@ -216,6 +221,7 @@ def ipa_to_rtgs(ipa: str) -> str: """ rtgs_parts = [] + ipa_cut = _ipa_cut() ipa_parts = ipa_cut.word_tokenize(ipa) for i, ipa_part in enumerate(ipa_parts): From 99c6d3cedef4452d7bc84059b4405a96581a0d93 Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 13:49:46 +0700 Subject: [PATCH 02/18] refactor: lazy load vowel tokenizer --- pythainlp/util/spell_words.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pythainlp/util/spell_words.py b/pythainlp/util/spell_words.py index 24b005e56..8ab43f0aa 100644 --- a/pythainlp/util/spell_words.py +++ b/pythainlp/util/spell_words.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from functools import lru_cache import re from typing import List @@ -48,7 +49,10 @@ for i in thai_below_vowels: dict_vowel[i] = "อ" + i -_cut = Tokenizer(list(dict_vowel.keys()) + list(thai_consonants), engine="mm") +@lru_cache +def _cut(): + """Lazy load vowel tokenizer with cache""" + return Tokenizer(list(dict_vowel.keys()) + list(thai_consonants), engine="mm") def _clean(w): @@ -93,7 +97,7 @@ def spell_syllable(text: str) -> List[str]: print(spell_syllable("แมว")) # output: ['มอ', 'วอ', 'แอ', 'แมว'] """ - tokens = _cut.word_tokenize(_clean(text)) + tokens = _cut().word_tokenize(_clean(text)) c_only = [tok + "อ" for tok in tokens if tok in set(thai_consonants)] v_only = [dict_vowel[tok] for tok in tokens if tok in set(dict_vowel)] From ab43b54a0660a6746637c1f50e510a6e25560789 Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 13:50:28 +0700 Subject: [PATCH 03/18] refactor: lazy load thai time cut --- pythainlp/util/time.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pythainlp/util/time.py b/pythainlp/util/time.py index ff54cb91a..85738b94a 100644 --- a/pythainlp/util/time.py +++ b/pythainlp/util/time.py @@ -8,6 +8,7 @@ Convert time string or time object to Thai words. """ from datetime import datetime, time +from functools import lru_cache from typing import Union from pythainlp.tokenize import Tokenizer @@ -43,9 +44,13 @@ "นาฬิกา": 0, "ครึ่ง": 30, } -_THAI_TIME_CUT = Tokenizer( - custom_dict=list(_DICT_THAI_TIME.keys()), engine="newmm" -) + +@lru_cache +def _thai_time_cut(): + """Lazy load Thai time tokenizer with cache""" + return Tokenizer( + custom_dict=list(_DICT_THAI_TIME.keys()), engine="newmm" + ) _THAI_TIME_AFFIX = [ "โมงเช้า", "บ่ายโมง", @@ -266,10 +271,10 @@ def thaiword_to_time(text: str, padding: bool = True) -> str: _LIST_THAI_TIME = _time.split("|") del _time - hour = _THAI_TIME_CUT.word_tokenize(_LIST_THAI_TIME[0]) + hour = _thai_time_cut().word_tokenize(_LIST_THAI_TIME[0]) minute = _LIST_THAI_TIME[1] if len(minute) > 1: - minute = _THAI_TIME_CUT.word_tokenize(minute) + minute = _thai_time_cut().word_tokenize(minute) else: minute = 0 text = "" From 6962fab423153b79fbe872cdc9152f22e952621a Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 13:51:17 +0700 Subject: [PATCH 04/18] refactor: lazy load numeral tokenizer --- pythainlp/util/wordtonum.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 2e08d3b6a..60c83fc96 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -8,6 +8,7 @@ First version of the code adapted from Korakot Chaovavanich's notebook https://colab.research.google.com/drive/148WNIeclf0kOU6QxKd6pcfwpSs8l-VKD#scrollTo=EuVDd0nNuI8Q """ +from functools import lru_cache import re from typing import List @@ -47,7 +48,10 @@ _valid_tokens = ( set(_digits.keys()) | set(_powers_of_10.keys()) | {"ล้าน", "ลบ"} ) -_tokenizer = Tokenizer(custom_dict=_valid_tokens) +@lru_cache +def _tokenizer(): + """Lazy load Thai numeral tokenizer with cache""" + return Tokenizer(custom_dict=_valid_tokens) def _check_is_thainum(word: str): @@ -96,7 +100,7 @@ def thaiword_to_num(word: str) -> int: if not _re_thai_numerals.fullmatch(word): raise ValueError("The input string is not a valid Thai numeral") - tokens = _tokenizer.word_tokenize(word) + tokens = _tokenizer().word_tokenize(word) accumulated = 0 next_digit = 1 From b006a79a51ce20b9bfc35095381d64fd6c5d875c Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 13:54:20 +0700 Subject: [PATCH 05/18] refactor: lazy load thai2fit tokenizer --- pythainlp/augment/word2vec/thai2fit.py | 5 +++-- pythainlp/tokenize/__init__.py | 9 ++------- pythainlp/tokenize/thai2fit.py | 10 ++++++++++ pythainlp/ulmfit/core.py | 9 ++++++--- pythainlp/ulmfit/tokenizer.py | 4 ++-- pythainlp/word_vector/core.py | 4 ++-- 6 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 pythainlp/tokenize/thai2fit.py diff --git a/pythainlp/augment/word2vec/thai2fit.py b/pythainlp/augment/word2vec/thai2fit.py index 6c7e3b4c0..16aae20c2 100644 --- a/pythainlp/augment/word2vec/thai2fit.py +++ b/pythainlp/augment/word2vec/thai2fit.py @@ -6,7 +6,7 @@ from pythainlp.augment.word2vec.core import Word2VecAug from pythainlp.corpus import get_corpus_path -from pythainlp.tokenize import THAI2FIT_TOKENIZER +from pythainlp.tokenize import thai2fit_tokenizer class Thai2fitAug: @@ -26,7 +26,8 @@ def tokenizer(self, text: str) -> List[str]: :param str text: Thai text :rtype: List[str] """ - return THAI2FIT_TOKENIZER.word_tokenize(text) + tok = thai2fit_tokenizer() + return tok.word_tokenize(text) def load_w2v(self): """ diff --git a/pythainlp/tokenize/__init__.py b/pythainlp/tokenize/__init__.py index 160996d7f..b5902af51 100644 --- a/pythainlp/tokenize/__init__.py +++ b/pythainlp/tokenize/__init__.py @@ -7,7 +7,7 @@ """ __all__ = [ - "THAI2FIT_TOKENIZER", + "thai2fit_tokenizer", "Tokenizer", "Trie", "paragraph_tokenize", @@ -41,9 +41,4 @@ word_tokenize, display_cell_tokenize, ) - -from pythainlp.corpus import get_corpus as _get_corpus - -THAI2FIT_TOKENIZER = Tokenizer( - custom_dict=_get_corpus("words_th_thai2fit_201810.txt"), engine="mm" -) +from pythainlp.tokenize.thai2fit import thai2fit_tokenizer diff --git a/pythainlp/tokenize/thai2fit.py b/pythainlp/tokenize/thai2fit.py new file mode 100644 index 000000000..0c0463888 --- /dev/null +++ b/pythainlp/tokenize/thai2fit.py @@ -0,0 +1,10 @@ +from functools import lru_cache +from pythainlp.corpus import get_corpus +from pythainlp.tokenize import Tokenizer + +@lru_cache +def thai2fit_tokenizer(): + """Lazy load Thai2Fit tokenizer with cache""" + return Tokenizer( + custom_dict=get_corpus("words_th_thai2fit_201810.txt"), engine="mm" + ) diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index 1494dcf1f..88f07e01c 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -12,7 +12,7 @@ import torch from pythainlp.corpus import get_corpus_path -from pythainlp.tokenize import THAI2FIT_TOKENIZER +from pythainlp.tokenize import thai2fit_tokenizer from pythainlp.ulmfit.preprocess import ( fix_html, lowercase_all, @@ -67,7 +67,7 @@ def process_thai( text: str, pre_rules: Collection = pre_rules_th_sparse, - tok_func: Callable = THAI2FIT_TOKENIZER.word_tokenize, + tok_func: Callable | None = None, post_rules: Collection = post_rules_th_sparse, ) -> Collection[str]: """ @@ -132,6 +132,9 @@ def process_thai( """ res = text + if tok_func is None: + tok_func = thai2fit_tokenizer().word_tokenize + for rule in pre_rules: res = rule(res) res = tok_func(res) @@ -183,7 +186,7 @@ def document_vector(text: str, learn, data, agg: str = "mean"): """ - s = THAI2FIT_TOKENIZER.word_tokenize(text) + s = thai2fit_tokenizer().word_tokenize(text) t = torch.tensor(data.vocab.numericalize(s), requires_grad=False).to( device ) diff --git a/pythainlp/ulmfit/tokenizer.py b/pythainlp/ulmfit/tokenizer.py index a732891e7..0156a4ff8 100644 --- a/pythainlp/ulmfit/tokenizer.py +++ b/pythainlp/ulmfit/tokenizer.py @@ -8,7 +8,7 @@ from typing import Collection, List -from pythainlp.tokenize import THAI2FIT_TOKENIZER +from pythainlp.tokenize import thai2fit_tokenizer class BaseTokenizer: @@ -65,7 +65,7 @@ def tokenizer(text: str) -> List[str]: ' ', 'ภาวนามยปัญญา'] """ - return THAI2FIT_TOKENIZER.word_tokenize(text) + return thai2fit_tokenizer().word_tokenize(text) def add_special_cases(self, toks): pass diff --git a/pythainlp/word_vector/core.py b/pythainlp/word_vector/core.py index 777c38df2..886c596f2 100644 --- a/pythainlp/word_vector/core.py +++ b/pythainlp/word_vector/core.py @@ -9,7 +9,7 @@ from numpy import ndarray, zeros from pythainlp.corpus import get_corpus_path -from pythainlp.tokenize import THAI2FIT_TOKENIZER, word_tokenize +from pythainlp.tokenize import thai2fit_tokenizer, word_tokenize WV_DIM = 300 # word vector dimension @@ -61,7 +61,7 @@ def load_wordvector(self, model_name: str): self.WV_DIM = self.model.vector_size if self.model_name == "thai2fit_wv": - self.tokenize = THAI2FIT_TOKENIZER.word_tokenize + self.tokenize = thai2fit_tokenizer().word_tokenize else: self.tokenize = word_tokenize From 9f18209d0432eb11501d4e236de3cf8091bee661 Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 13:56:54 +0700 Subject: [PATCH 06/18] refactor: lazy load thaiwords tokenizer --- pythainlp/util/wordtonum.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 60c83fc96..f88bf7458 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -68,7 +68,10 @@ def _check_is_thainum(word: str): _dict_words += list(_digits.keys()) _dict_words += ["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"] -_tokenizer_thaiwords = Tokenizer(_dict_words) +@lru_cache +def _tokenizer_thaiwords(): + """Lazy load Thai words tokenizer with cache""" + return Tokenizer(_dict_words) def thaiword_to_num(word: str) -> int: From 3b3bf6cdf8acc6049db5ac208c355baed4c38c51 Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 14:00:00 +0700 Subject: [PATCH 07/18] refactor: lazy load trie --- pythainlp/tokenize/core.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index f3d316a2e..c7af53861 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -97,7 +97,7 @@ def word_detokenize( def word_tokenize( text: str, - custom_dict: Trie = Trie([]), + custom_dict: Trie | None = None, engine: str = DEFAULT_WORD_TOKENIZE_ENGINE, keep_whitespace: bool = True, join_broken_num: bool = True, @@ -223,6 +223,9 @@ def word_tokenize( segments = [] + if custom_dict is None: + custom_dict = Trie([]) + if custom_dict and engine in ( "attacut", "icu", @@ -877,6 +880,7 @@ def __init__( :param bool keep_whitespace: True to keep whitespace, a common mark for end of phrase in Thai """ + print("HEHE") self.__trie_dict = Trie([]) if custom_dict: self.__trie_dict = dict_trie(custom_dict) From 3427286e5fe2f3c6a5b9dc654e028ea9d17cf8ff Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 14:11:57 +0700 Subject: [PATCH 08/18] refactor: lazy load word dict trie and syllable dict trie --- pythainlp/tokenize/__init__.py | 13 ++++++++++--- pythainlp/tokenize/core.py | 8 ++++---- pythainlp/tokenize/longest.py | 8 ++++---- pythainlp/tokenize/multi_cut.py | 23 ++++++++++++++--------- pythainlp/tokenize/newmm.py | 10 +++++----- tests/core/test_tokenize.py | 4 ++-- tests/extra/testx_tokenize.py | 6 +++--- 7 files changed, 42 insertions(+), 30 deletions(-) diff --git a/pythainlp/tokenize/__init__.py b/pythainlp/tokenize/__init__.py index b5902af51..8c64cecc0 100644 --- a/pythainlp/tokenize/__init__.py +++ b/pythainlp/tokenize/__init__.py @@ -19,6 +19,7 @@ "display_cell_tokenize", ] +from functools import lru_cache from pythainlp.corpus import thai_syllables, thai_words from pythainlp.util.trie import Trie @@ -27,9 +28,15 @@ DEFAULT_SUBWORD_TOKENIZE_ENGINE = "tcc" DEFAULT_SYLLABLE_TOKENIZE_ENGINE = "han_solo" -DEFAULT_WORD_DICT_TRIE = Trie(thai_words()) -DEFAULT_SYLLABLE_DICT_TRIE = Trie(thai_syllables()) -DEFAULT_DICT_TRIE = DEFAULT_WORD_DICT_TRIE +@lru_cache +def word_dict_trie(): + """Lazy load default word dict trie with cache""" + return Trie(thai_words()) + +@lru_cache +def syllable_dict_trie(): + """Lazy load default syllable dict trie with cache""" + return Trie(thai_syllables()) from pythainlp.tokenize.core import ( Tokenizer, diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index c7af53861..8debcd3b1 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -13,9 +13,9 @@ from pythainlp.tokenize import ( DEFAULT_SENT_TOKENIZE_ENGINE, DEFAULT_SUBWORD_TOKENIZE_ENGINE, - DEFAULT_SYLLABLE_DICT_TRIE, + syllable_dict_trie, DEFAULT_SYLLABLE_TOKENIZE_ENGINE, - DEFAULT_WORD_DICT_TRIE, + word_dict_trie, DEFAULT_WORD_TOKENIZE_ENGINE, ) from pythainlp.tokenize._utils import ( @@ -693,7 +693,7 @@ def subword_tokenize( for word in words: segments.extend( word_tokenize( - text=word, custom_dict=DEFAULT_SYLLABLE_DICT_TRIE + text=word, custom_dict=syllable_dict_trie() ) ) elif engine == "ssg": @@ -885,7 +885,7 @@ def __init__( if custom_dict: self.__trie_dict = dict_trie(custom_dict) else: - self.__trie_dict = DEFAULT_WORD_DICT_TRIE + self.__trie_dict = word_dict_trie() self.__engine = engine if self.__engine not in ["newmm", "mm", "longest", "deepcut"]: raise NotImplementedError( diff --git a/pythainlp/tokenize/longest.py b/pythainlp/tokenize/longest.py index f4cfc571d..969f47271 100644 --- a/pythainlp/tokenize/longest.py +++ b/pythainlp/tokenize/longest.py @@ -12,10 +12,10 @@ """ import re -from typing import Dict, List, Union +from typing import Dict, List, Optional, Union from pythainlp import thai_tonemarks -from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE +from pythainlp.tokenize import word_dict_trie from pythainlp.util import Trie _FRONT_DEP_CHAR = [ @@ -152,7 +152,7 @@ def tokenize(self, text: str) -> List[str]: _tokenizers: Dict[int, LongestMatchTokenizer] = {} -def segment(text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE) -> List[str]: +def segment(text: str, custom_dict: Optional[Trie] = None) -> List[str]: """ Dictionary-based longest matching word segmentation. @@ -164,7 +164,7 @@ def segment(text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE) -> List[str]: return [] if not custom_dict: - custom_dict = DEFAULT_WORD_DICT_TRIE + custom_dict = word_dict_trie() global _tokenizers custom_dict_ref_id = id(custom_dict) diff --git a/pythainlp/tokenize/multi_cut.py b/pythainlp/tokenize/multi_cut.py index afd57e88a..5f63a07ad 100644 --- a/pythainlp/tokenize/multi_cut.py +++ b/pythainlp/tokenize/multi_cut.py @@ -15,9 +15,9 @@ import re from collections import defaultdict -from typing import Iterator, List +from typing import Iterator, List, Optional -from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE +from pythainlp.tokenize import word_dict_trie from pythainlp.util import Trie @@ -48,12 +48,11 @@ def __init__(self, value, multi=None, in_dict=True): def _multicut( - text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE + text: str, custom_dict: Optional[Trie] = None ) -> Iterator[LatticeString]: """Return LatticeString""" if not custom_dict: - custom_dict = DEFAULT_WORD_DICT_TRIE - + custom_dict = word_dict_trie() len_text = len(text) words_at = defaultdict(list) # main data structure @@ -123,39 +122,45 @@ def _combine(ww: List[LatticeString]) -> Iterator[str]: def segment( - text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE + text: str, custom_dict: Optional[Trie] = None ) -> List[str]: """Dictionary-based maximum matching word segmentation. :param text: text to be tokenized :type text: str :param custom_dict: tokenization dictionary,\ - defaults to DEFAULT_WORD_DICT_TRIE + defaults to word_dict_trie() :type custom_dict: Trie, optional :return: list of segmented tokens :rtype: List[str] """ if not text or not isinstance(text, str): return [] + + if not custom_dict: + custom_dict = word_dict_trie() return list(_multicut(text, custom_dict=custom_dict)) def find_all_segment( - text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE + text: str, custom_dict: Optional[Trie] = None ) -> List[str]: """Get all possible segment variations. :param text: input string to be tokenized :type text: str :param custom_dict: tokenization dictionary,\ - defaults to DEFAULT_WORD_DICT_TRIE + defaults to word_dict_trie() :type custom_dict: Trie, optional :return: list of segment variations :rtype: List[str] """ if not text or not isinstance(text, str): return [] + + if not custom_dict: + custom_dict = word_dict_trie() ww = list(_multicut(text, custom_dict=custom_dict)) diff --git a/pythainlp/tokenize/newmm.py b/pythainlp/tokenize/newmm.py index 935da7453..ec30356e5 100644 --- a/pythainlp/tokenize/newmm.py +++ b/pythainlp/tokenize/newmm.py @@ -18,9 +18,9 @@ import re from collections import defaultdict from heapq import heappop, heappush -from typing import Generator, List +from typing import Generator, List, Optional -from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE +from pythainlp.tokenize import word_dict_trie from pythainlp.tokenize.tcc_p import tcc_pos from pythainlp.util import Trie @@ -140,7 +140,7 @@ def _onecut(text: str, custom_dict: Trie) -> Generator[str, None, None]: def segment( text: str, - custom_dict: Trie = DEFAULT_WORD_DICT_TRIE, + custom_dict: Optional[Trie] = None, safe_mode: bool = False, ) -> List[str]: """Maximal-matching word segmentation constrained by Thai Character Cluster. @@ -153,7 +153,7 @@ def segment( :param text: text to be tokenized :type text: str :param custom_dict: tokenization dictionary,\ - defaults to DEFAULT_WORD_DICT_TRIE + defaults to word_dict_trie() :type custom_dict: Trie, optional :param safe_mode: reduce chance for long processing time for long text\ with many ambiguous breaking points, defaults to False @@ -165,7 +165,7 @@ def segment( return [] if not custom_dict: - custom_dict = DEFAULT_WORD_DICT_TRIE + custom_dict = word_dict_trie() if not safe_mode or len(text) < _TEXT_SCAN_END: return list(_onecut(text, custom_dict)) diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py index bd5562eb3..e5672506d 100644 --- a/tests/core/test_tokenize.py +++ b/tests/core/test_tokenize.py @@ -6,7 +6,7 @@ import unittest from pythainlp.tokenize import ( - DEFAULT_WORD_DICT_TRIE, + word_dict_trie, Tokenizer, etcc, longest, @@ -261,7 +261,7 @@ def test_numeric_data_format(self): class TokenizeTestCase(unittest.TestCase): def test_Tokenizer(self): - _tokenizer = Tokenizer(DEFAULT_WORD_DICT_TRIE) + _tokenizer = Tokenizer(word_dict_trie()) self.assertEqual(_tokenizer.word_tokenize(""), []) _tokenizer.set_tokenize_engine("longest") self.assertEqual(_tokenizer.word_tokenize(None), []) diff --git a/tests/extra/testx_tokenize.py b/tests/extra/testx_tokenize.py index 321c7c747..e24e7eeb6 100644 --- a/tests/extra/testx_tokenize.py +++ b/tests/extra/testx_tokenize.py @@ -8,7 +8,7 @@ import unittest from pythainlp.tokenize import ( - DEFAULT_WORD_DICT_TRIE, + word_dict_trie, attacut, deepcut, nercut, @@ -275,12 +275,12 @@ def test_word_tokenize_deepcut(self): def test_deepcut(self): self.assertEqual(deepcut.segment(None), []) self.assertEqual(deepcut.segment(""), []) - self.assertIsNotNone(deepcut.segment("ทดสอบ", DEFAULT_WORD_DICT_TRIE)) + self.assertIsNotNone(deepcut.segment("ทดสอบ", word_dict_trie())) self.assertIsNotNone(deepcut.segment("ทดสอบ", ["ทด", "สอบ"])) self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut")) self.assertIsNotNone( word_tokenize( - "ทดสอบ", engine="deepcut", custom_dict=DEFAULT_WORD_DICT_TRIE + "ทดสอบ", engine="deepcut", custom_dict=word_dict_trie() ) ) From e04c6e3657acb7b084d3c07152c6d2a31af2f022 Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 14:31:49 +0700 Subject: [PATCH 09/18] fix: use older typehint --- pythainlp/tokenize/core.py | 4 ++-- pythainlp/ulmfit/core.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index 8debcd3b1..9332d2710 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -8,7 +8,7 @@ import copy import re -from typing import Iterable, List, Union +from typing import Iterable, List, Optional, Union from pythainlp.tokenize import ( DEFAULT_SENT_TOKENIZE_ENGINE, @@ -97,7 +97,7 @@ def word_detokenize( def word_tokenize( text: str, - custom_dict: Trie | None = None, + custom_dict: Optional[Trie] = None, engine: str = DEFAULT_WORD_TOKENIZE_ENGINE, keep_whitespace: bool = True, join_broken_num: bool = True, diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index 88f07e01c..760523802 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -6,7 +6,7 @@ Universal Language Model Fine-tuning for Text Classification (ULMFiT). """ import collections -from typing import Callable, Collection +from typing import Callable, Collection, Optional import numpy as np import torch @@ -67,7 +67,7 @@ def process_thai( text: str, pre_rules: Collection = pre_rules_th_sparse, - tok_func: Callable | None = None, + tok_func: Optional[Callable] = None, post_rules: Collection = post_rules_th_sparse, ) -> Collection[str]: """ From 915fff98e0f9fba81c0ece01d157c8a70aa0a33b Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 14:32:16 +0700 Subject: [PATCH 10/18] fix: clean up debug comment --- pythainlp/tokenize/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index 9332d2710..a6cdaf737 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -880,7 +880,6 @@ def __init__( :param bool keep_whitespace: True to keep whitespace, a common mark for end of phrase in Thai """ - print("HEHE") self.__trie_dict = Trie([]) if custom_dict: self.__trie_dict = dict_trie(custom_dict) From b78fc0a60bbc5df8b218323e7b96c6d7d84a1bac Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 14:40:31 +0700 Subject: [PATCH 11/18] chore: remove trailing space --- pythainlp/tokenize/multi_cut.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pythainlp/tokenize/multi_cut.py b/pythainlp/tokenize/multi_cut.py index 5f63a07ad..3576ae9cc 100644 --- a/pythainlp/tokenize/multi_cut.py +++ b/pythainlp/tokenize/multi_cut.py @@ -136,7 +136,7 @@ def segment( """ if not text or not isinstance(text, str): return [] - + if not custom_dict: custom_dict = word_dict_trie() @@ -158,7 +158,7 @@ def find_all_segment( """ if not text or not isinstance(text, str): return [] - + if not custom_dict: custom_dict = word_dict_trie() From f31c0110cc5c71865d1b5a16b50d40fda352b40b Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 14:43:07 +0700 Subject: [PATCH 12/18] fix: add license header to thai2fit.py --- pythainlp/tokenize/thai2fit.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pythainlp/tokenize/thai2fit.py b/pythainlp/tokenize/thai2fit.py index 0c0463888..7637d22d5 100644 --- a/pythainlp/tokenize/thai2fit.py +++ b/pythainlp/tokenize/thai2fit.py @@ -1,3 +1,8 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + from functools import lru_cache from pythainlp.corpus import get_corpus from pythainlp.tokenize import Tokenizer From 22d239a50204cf526b48d504227bd97b48887294 Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 14:52:23 +0700 Subject: [PATCH 13/18] refactor: move dict words to thaiwords tokenizer --- pythainlp/util/wordtonum.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index f88bf7458..4fc263c99 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -64,13 +64,12 @@ def _check_is_thainum(word: str): return (False, None) -_dict_words = [i for i in list(thai_words()) if not _check_is_thainum(i)[0]] -_dict_words += list(_digits.keys()) -_dict_words += ["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"] - @lru_cache def _tokenizer_thaiwords(): """Lazy load Thai words tokenizer with cache""" + _dict_words = [i for i in list(thai_words()) if not _check_is_thainum(i)[0]] + _dict_words += list(_digits.keys()) + _dict_words += ["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"] return Tokenizer(_dict_words) From 9e58a993dd64975c05696643c4e2203a2cb1581e Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 14:56:48 +0700 Subject: [PATCH 14/18] refactor: lazy load etcc tokenizer --- pythainlp/tokenize/etcc.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pythainlp/tokenize/etcc.py b/pythainlp/tokenize/etcc.py index 8d8769515..557be3cef 100644 --- a/pythainlp/tokenize/etcc.py +++ b/pythainlp/tokenize/etcc.py @@ -19,6 +19,7 @@ and backward longest matching techniques." In International Symposium on Communications and Information Technology (ISCIT), pp. 37-40. 2001. """ +from functools import lru_cache import re from typing import List @@ -26,7 +27,11 @@ from pythainlp.corpus import get_corpus from pythainlp.tokenize import Tokenizer -_cut_etcc = Tokenizer(get_corpus("etcc.txt"), engine="longest") +@lru_cache +def _cut_etcc(): + """Lazy load ETCC tokenizer with cache""" + return Tokenizer(get_corpus("etcc.txt"), engine="longest") + _PAT_ENDING_CHAR = f"[{thai_follow_vowels}ๆฯ]" _RE_ENDING_CHAR = re.compile(_PAT_ENDING_CHAR) @@ -64,4 +69,4 @@ def segment(text: str) -> List[str]: if not text or not isinstance(text, str): return [] - return _cut_subword(_cut_etcc.word_tokenize(text)) + return _cut_subword(_cut_etcc().word_tokenize(text)) From 3f10ffa3937b14b7e552a20dd3b53e3e00e15d95 Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 15:04:42 +0700 Subject: [PATCH 15/18] refactor: lazy load spell checker --- docs/api/spell.rst | 4 ++-- pythainlp/spell/__init__.py | 2 +- pythainlp/spell/core.py | 9 +++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/api/spell.rst b/docs/api/spell.rst index ce2dd035d..b0345219d 100644 --- a/docs/api/spell.rst +++ b/docs/api/spell.rst @@ -48,9 +48,9 @@ The `NorvigSpellChecker` class is a fundamental component of the `pythainlp.spel DEFAULT_SPELL_CHECKER ~~~~~~~~~~~~~~~~~~~~~ .. autodata:: DEFAULT_SPELL_CHECKER - :annotation: = Default instance of the standard NorvigSpellChecker, using word list data from the Thai National Corpus: http://www.arts.chula.ac.th/ling/tnc/ + :annotation: = Default reference of the standard NorvigSpellChecker, using word list data from the Thai National Corpus: http://www.arts.chula.ac.th/ling/tnc/ -The `DEFAULT_SPELL_CHECKER` is an instance of the `NorvigSpellChecker` class with default settings. It is pre-configured to use word list data from the Thai National Corpus, making it a reliable choice for general spell-checking tasks. +The `DEFAULT_SPELL_CHECKER` is an reference to the `NorvigSpellChecker` class with default settings. It is pre-configured to use word list data from the Thai National Corpus, making it a reliable choice for general spell-checking tasks. References ---------- diff --git a/pythainlp/spell/__init__.py b/pythainlp/spell/__init__.py index 33bee579e..54e9af445 100644 --- a/pythainlp/spell/__init__.py +++ b/pythainlp/spell/__init__.py @@ -18,7 +18,7 @@ from pythainlp.spell.pn import NorvigSpellChecker -DEFAULT_SPELL_CHECKER = NorvigSpellChecker() +DEFAULT_SPELL_CHECKER = NorvigSpellChecker # these imports are placed here to avoid circular imports from pythainlp.spell.core import correct, correct_sent, spell, spell_sent diff --git a/pythainlp/spell/core.py b/pythainlp/spell/core.py index 8c874d5ce..a9320e756 100644 --- a/pythainlp/spell/core.py +++ b/pythainlp/spell/core.py @@ -6,11 +6,16 @@ Spell checking functions """ +from functools import lru_cache import itertools from typing import List from pythainlp.spell import DEFAULT_SPELL_CHECKER +@lru_cache +def default_spell_checker(): + """Lazy load default spell checker with cache""" + return DEFAULT_SPELL_CHECKER() def spell(word: str, engine: str = "pn") -> List[str]: """ @@ -72,7 +77,7 @@ def spell(word: str, engine: str = "pn") -> List[str]: text_correct = SPELL_CHECKER(word) else: - text_correct = DEFAULT_SPELL_CHECKER.spell(word) + text_correct = default_spell_checker().spell(word) return text_correct @@ -125,7 +130,7 @@ def correct(word: str, engine: str = "pn") -> str: text_correct = SPELL_CHECKER(word) else: - text_correct = DEFAULT_SPELL_CHECKER.correct(word) + text_correct = default_spell_checker().correct(word) return text_correct From 66bbeec053658f1e38e04f8aa2206822a40622d8 Mon Sep 17 00:00:00 2001 From: what_in_the_nim Date: Sat, 10 Jan 2026 16:13:25 +0700 Subject: [PATCH 16/18] fix: forget to call func --- pythainlp/util/wordtonum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 4fc263c99..6b320b41e 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -191,7 +191,7 @@ def text_to_num(text: str) -> List[str]: # output: ['10021889', 'บาท'] """ - _temp = _tokenizer_thaiwords.word_tokenize(text) + _temp = _tokenizer_thaiwords().word_tokenize(text) thainum = [] last_index = -1 list_word_new = [] From c1f0770ddc5c4ff52c85cc826c6a0bc155dd8e3a Mon Sep 17 00:00:00 2001 From: Chatcharin Sangbutsarakum <67754293+what-in-the-nim@users.noreply.github.com> Date: Sat, 10 Jan 2026 20:00:26 +0700 Subject: [PATCH 17/18] chore: update docstring Co-authored-by: Arthit Suriyawongkul --- pythainlp/tokenize/multi_cut.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythainlp/tokenize/multi_cut.py b/pythainlp/tokenize/multi_cut.py index 3576ae9cc..2ac0b0b49 100644 --- a/pythainlp/tokenize/multi_cut.py +++ b/pythainlp/tokenize/multi_cut.py @@ -129,7 +129,7 @@ def segment( :param text: text to be tokenized :type text: str :param custom_dict: tokenization dictionary,\ - defaults to word_dict_trie() + defaults to a Trie generated from pythainlp.corpus.thai_words :type custom_dict: Trie, optional :return: list of segmented tokens :rtype: List[str] From f91a9776b61535b19263721a5377c6a895ac930d Mon Sep 17 00:00:00 2001 From: Chatcharin Sangbutsarakum <67754293+what-in-the-nim@users.noreply.github.com> Date: Sat, 10 Jan 2026 20:41:14 +0700 Subject: [PATCH 18/18] chore: update license header Co-authored-by: Arthit Suriyawongkul --- pythainlp/tokenize/thai2fit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythainlp/tokenize/thai2fit.py b/pythainlp/tokenize/thai2fit.py index 7637d22d5..31556ae2e 100644 --- a/pythainlp/tokenize/thai2fit.py +++ b/pythainlp/tokenize/thai2fit.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0