diff --git a/docs/api/spell.rst b/docs/api/spell.rst index ce2dd035d..b0345219d 100644 --- a/docs/api/spell.rst +++ b/docs/api/spell.rst @@ -48,9 +48,9 @@ The `NorvigSpellChecker` class is a fundamental component of the `pythainlp.spel DEFAULT_SPELL_CHECKER ~~~~~~~~~~~~~~~~~~~~~ .. autodata:: DEFAULT_SPELL_CHECKER - :annotation: = Default instance of the standard NorvigSpellChecker, using word list data from the Thai National Corpus: http://www.arts.chula.ac.th/ling/tnc/ + :annotation: = Default reference of the standard NorvigSpellChecker, using word list data from the Thai National Corpus: http://www.arts.chula.ac.th/ling/tnc/ -The `DEFAULT_SPELL_CHECKER` is an instance of the `NorvigSpellChecker` class with default settings. It is pre-configured to use word list data from the Thai National Corpus, making it a reliable choice for general spell-checking tasks. +The `DEFAULT_SPELL_CHECKER` is an reference to the `NorvigSpellChecker` class with default settings. It is pre-configured to use word list data from the Thai National Corpus, making it a reliable choice for general spell-checking tasks. References ---------- diff --git a/pythainlp/augment/word2vec/thai2fit.py b/pythainlp/augment/word2vec/thai2fit.py index 6c7e3b4c0..16aae20c2 100644 --- a/pythainlp/augment/word2vec/thai2fit.py +++ b/pythainlp/augment/word2vec/thai2fit.py @@ -6,7 +6,7 @@ from pythainlp.augment.word2vec.core import Word2VecAug from pythainlp.corpus import get_corpus_path -from pythainlp.tokenize import THAI2FIT_TOKENIZER +from pythainlp.tokenize import thai2fit_tokenizer class Thai2fitAug: @@ -26,7 +26,8 @@ def tokenizer(self, text: str) -> List[str]: :param str text: Thai text :rtype: List[str] """ - return THAI2FIT_TOKENIZER.word_tokenize(text) + tok = thai2fit_tokenizer() + return tok.word_tokenize(text) def load_w2v(self): """ diff --git a/pythainlp/spell/__init__.py b/pythainlp/spell/__init__.py index 33bee579e..54e9af445 100644 --- a/pythainlp/spell/__init__.py +++ b/pythainlp/spell/__init__.py @@ -18,7 +18,7 @@ from pythainlp.spell.pn import NorvigSpellChecker -DEFAULT_SPELL_CHECKER = NorvigSpellChecker() +DEFAULT_SPELL_CHECKER = NorvigSpellChecker # these imports are placed here to avoid circular imports from pythainlp.spell.core import correct, correct_sent, spell, spell_sent diff --git a/pythainlp/spell/core.py b/pythainlp/spell/core.py index 8c874d5ce..a9320e756 100644 --- a/pythainlp/spell/core.py +++ b/pythainlp/spell/core.py @@ -6,11 +6,16 @@ Spell checking functions """ +from functools import lru_cache import itertools from typing import List from pythainlp.spell import DEFAULT_SPELL_CHECKER +@lru_cache +def default_spell_checker(): + """Lazy load default spell checker with cache""" + return DEFAULT_SPELL_CHECKER() def spell(word: str, engine: str = "pn") -> List[str]: """ @@ -72,7 +77,7 @@ def spell(word: str, engine: str = "pn") -> List[str]: text_correct = SPELL_CHECKER(word) else: - text_correct = DEFAULT_SPELL_CHECKER.spell(word) + text_correct = default_spell_checker().spell(word) return text_correct @@ -125,7 +130,7 @@ def correct(word: str, engine: str = "pn") -> str: text_correct = SPELL_CHECKER(word) else: - text_correct = DEFAULT_SPELL_CHECKER.correct(word) + text_correct = default_spell_checker().correct(word) return text_correct diff --git a/pythainlp/tokenize/__init__.py b/pythainlp/tokenize/__init__.py index 160996d7f..8c64cecc0 100644 --- a/pythainlp/tokenize/__init__.py +++ b/pythainlp/tokenize/__init__.py @@ -7,7 +7,7 @@ """ __all__ = [ - "THAI2FIT_TOKENIZER", + "thai2fit_tokenizer", "Tokenizer", "Trie", "paragraph_tokenize", @@ -19,6 +19,7 @@ "display_cell_tokenize", ] +from functools import lru_cache from pythainlp.corpus import thai_syllables, thai_words from pythainlp.util.trie import Trie @@ -27,9 +28,15 @@ DEFAULT_SUBWORD_TOKENIZE_ENGINE = "tcc" DEFAULT_SYLLABLE_TOKENIZE_ENGINE = "han_solo" -DEFAULT_WORD_DICT_TRIE = Trie(thai_words()) -DEFAULT_SYLLABLE_DICT_TRIE = Trie(thai_syllables()) -DEFAULT_DICT_TRIE = DEFAULT_WORD_DICT_TRIE +@lru_cache +def word_dict_trie(): + """Lazy load default word dict trie with cache""" + return Trie(thai_words()) + +@lru_cache +def syllable_dict_trie(): + """Lazy load default syllable dict trie with cache""" + return Trie(thai_syllables()) from pythainlp.tokenize.core import ( Tokenizer, @@ -41,9 +48,4 @@ word_tokenize, display_cell_tokenize, ) - -from pythainlp.corpus import get_corpus as _get_corpus - -THAI2FIT_TOKENIZER = Tokenizer( - custom_dict=_get_corpus("words_th_thai2fit_201810.txt"), engine="mm" -) +from pythainlp.tokenize.thai2fit import thai2fit_tokenizer diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index f3d316a2e..a6cdaf737 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -8,14 +8,14 @@ import copy import re -from typing import Iterable, List, Union +from typing import Iterable, List, Optional, Union from pythainlp.tokenize import ( DEFAULT_SENT_TOKENIZE_ENGINE, DEFAULT_SUBWORD_TOKENIZE_ENGINE, - DEFAULT_SYLLABLE_DICT_TRIE, + syllable_dict_trie, DEFAULT_SYLLABLE_TOKENIZE_ENGINE, - DEFAULT_WORD_DICT_TRIE, + word_dict_trie, DEFAULT_WORD_TOKENIZE_ENGINE, ) from pythainlp.tokenize._utils import ( @@ -97,7 +97,7 @@ def word_detokenize( def word_tokenize( text: str, - custom_dict: Trie = Trie([]), + custom_dict: Optional[Trie] = None, engine: str = DEFAULT_WORD_TOKENIZE_ENGINE, keep_whitespace: bool = True, join_broken_num: bool = True, @@ -223,6 +223,9 @@ def word_tokenize( segments = [] + if custom_dict is None: + custom_dict = Trie([]) + if custom_dict and engine in ( "attacut", "icu", @@ -690,7 +693,7 @@ def subword_tokenize( for word in words: segments.extend( word_tokenize( - text=word, custom_dict=DEFAULT_SYLLABLE_DICT_TRIE + text=word, custom_dict=syllable_dict_trie() ) ) elif engine == "ssg": @@ -881,7 +884,7 @@ def __init__( if custom_dict: self.__trie_dict = dict_trie(custom_dict) else: - self.__trie_dict = DEFAULT_WORD_DICT_TRIE + self.__trie_dict = word_dict_trie() self.__engine = engine if self.__engine not in ["newmm", "mm", "longest", "deepcut"]: raise NotImplementedError( diff --git a/pythainlp/tokenize/etcc.py b/pythainlp/tokenize/etcc.py index 8d8769515..557be3cef 100644 --- a/pythainlp/tokenize/etcc.py +++ b/pythainlp/tokenize/etcc.py @@ -19,6 +19,7 @@ and backward longest matching techniques." In International Symposium on Communications and Information Technology (ISCIT), pp. 37-40. 2001. """ +from functools import lru_cache import re from typing import List @@ -26,7 +27,11 @@ from pythainlp.corpus import get_corpus from pythainlp.tokenize import Tokenizer -_cut_etcc = Tokenizer(get_corpus("etcc.txt"), engine="longest") +@lru_cache +def _cut_etcc(): + """Lazy load ETCC tokenizer with cache""" + return Tokenizer(get_corpus("etcc.txt"), engine="longest") + _PAT_ENDING_CHAR = f"[{thai_follow_vowels}ๆฯ]" _RE_ENDING_CHAR = re.compile(_PAT_ENDING_CHAR) @@ -64,4 +69,4 @@ def segment(text: str) -> List[str]: if not text or not isinstance(text, str): return [] - return _cut_subword(_cut_etcc.word_tokenize(text)) + return _cut_subword(_cut_etcc().word_tokenize(text)) diff --git a/pythainlp/tokenize/longest.py b/pythainlp/tokenize/longest.py index f4cfc571d..969f47271 100644 --- a/pythainlp/tokenize/longest.py +++ b/pythainlp/tokenize/longest.py @@ -12,10 +12,10 @@ """ import re -from typing import Dict, List, Union +from typing import Dict, List, Optional, Union from pythainlp import thai_tonemarks -from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE +from pythainlp.tokenize import word_dict_trie from pythainlp.util import Trie _FRONT_DEP_CHAR = [ @@ -152,7 +152,7 @@ def tokenize(self, text: str) -> List[str]: _tokenizers: Dict[int, LongestMatchTokenizer] = {} -def segment(text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE) -> List[str]: +def segment(text: str, custom_dict: Optional[Trie] = None) -> List[str]: """ Dictionary-based longest matching word segmentation. @@ -164,7 +164,7 @@ def segment(text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE) -> List[str]: return [] if not custom_dict: - custom_dict = DEFAULT_WORD_DICT_TRIE + custom_dict = word_dict_trie() global _tokenizers custom_dict_ref_id = id(custom_dict) diff --git a/pythainlp/tokenize/multi_cut.py b/pythainlp/tokenize/multi_cut.py index afd57e88a..2ac0b0b49 100644 --- a/pythainlp/tokenize/multi_cut.py +++ b/pythainlp/tokenize/multi_cut.py @@ -15,9 +15,9 @@ import re from collections import defaultdict -from typing import Iterator, List +from typing import Iterator, List, Optional -from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE +from pythainlp.tokenize import word_dict_trie from pythainlp.util import Trie @@ -48,12 +48,11 @@ def __init__(self, value, multi=None, in_dict=True): def _multicut( - text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE + text: str, custom_dict: Optional[Trie] = None ) -> Iterator[LatticeString]: """Return LatticeString""" if not custom_dict: - custom_dict = DEFAULT_WORD_DICT_TRIE - + custom_dict = word_dict_trie() len_text = len(text) words_at = defaultdict(list) # main data structure @@ -123,14 +122,14 @@ def _combine(ww: List[LatticeString]) -> Iterator[str]: def segment( - text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE + text: str, custom_dict: Optional[Trie] = None ) -> List[str]: """Dictionary-based maximum matching word segmentation. :param text: text to be tokenized :type text: str :param custom_dict: tokenization dictionary,\ - defaults to DEFAULT_WORD_DICT_TRIE + defaults to a Trie generated from pythainlp.corpus.thai_words :type custom_dict: Trie, optional :return: list of segmented tokens :rtype: List[str] @@ -138,18 +137,21 @@ def segment( if not text or not isinstance(text, str): return [] + if not custom_dict: + custom_dict = word_dict_trie() + return list(_multicut(text, custom_dict=custom_dict)) def find_all_segment( - text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE + text: str, custom_dict: Optional[Trie] = None ) -> List[str]: """Get all possible segment variations. :param text: input string to be tokenized :type text: str :param custom_dict: tokenization dictionary,\ - defaults to DEFAULT_WORD_DICT_TRIE + defaults to word_dict_trie() :type custom_dict: Trie, optional :return: list of segment variations :rtype: List[str] @@ -157,6 +159,9 @@ def find_all_segment( if not text or not isinstance(text, str): return [] + if not custom_dict: + custom_dict = word_dict_trie() + ww = list(_multicut(text, custom_dict=custom_dict)) return list(_combine(ww)) diff --git a/pythainlp/tokenize/newmm.py b/pythainlp/tokenize/newmm.py index 935da7453..ec30356e5 100644 --- a/pythainlp/tokenize/newmm.py +++ b/pythainlp/tokenize/newmm.py @@ -18,9 +18,9 @@ import re from collections import defaultdict from heapq import heappop, heappush -from typing import Generator, List +from typing import Generator, List, Optional -from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE +from pythainlp.tokenize import word_dict_trie from pythainlp.tokenize.tcc_p import tcc_pos from pythainlp.util import Trie @@ -140,7 +140,7 @@ def _onecut(text: str, custom_dict: Trie) -> Generator[str, None, None]: def segment( text: str, - custom_dict: Trie = DEFAULT_WORD_DICT_TRIE, + custom_dict: Optional[Trie] = None, safe_mode: bool = False, ) -> List[str]: """Maximal-matching word segmentation constrained by Thai Character Cluster. @@ -153,7 +153,7 @@ def segment( :param text: text to be tokenized :type text: str :param custom_dict: tokenization dictionary,\ - defaults to DEFAULT_WORD_DICT_TRIE + defaults to word_dict_trie() :type custom_dict: Trie, optional :param safe_mode: reduce chance for long processing time for long text\ with many ambiguous breaking points, defaults to False @@ -165,7 +165,7 @@ def segment( return [] if not custom_dict: - custom_dict = DEFAULT_WORD_DICT_TRIE + custom_dict = word_dict_trie() if not safe_mode or len(text) < _TEXT_SCAN_END: return list(_onecut(text, custom_dict)) diff --git a/pythainlp/tokenize/thai2fit.py b/pythainlp/tokenize/thai2fit.py new file mode 100644 index 000000000..31556ae2e --- /dev/null +++ b/pythainlp/tokenize/thai2fit.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# SPDX-FileCopyrightText: 2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 + +from functools import lru_cache +from pythainlp.corpus import get_corpus +from pythainlp.tokenize import Tokenizer + +@lru_cache +def thai2fit_tokenizer(): + """Lazy load Thai2Fit tokenizer with cache""" + return Tokenizer( + custom_dict=get_corpus("words_th_thai2fit_201810.txt"), engine="mm" + ) diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index 1494dcf1f..760523802 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -6,13 +6,13 @@ Universal Language Model Fine-tuning for Text Classification (ULMFiT). """ import collections -from typing import Callable, Collection +from typing import Callable, Collection, Optional import numpy as np import torch from pythainlp.corpus import get_corpus_path -from pythainlp.tokenize import THAI2FIT_TOKENIZER +from pythainlp.tokenize import thai2fit_tokenizer from pythainlp.ulmfit.preprocess import ( fix_html, lowercase_all, @@ -67,7 +67,7 @@ def process_thai( text: str, pre_rules: Collection = pre_rules_th_sparse, - tok_func: Callable = THAI2FIT_TOKENIZER.word_tokenize, + tok_func: Optional[Callable] = None, post_rules: Collection = post_rules_th_sparse, ) -> Collection[str]: """ @@ -132,6 +132,9 @@ def process_thai( """ res = text + if tok_func is None: + tok_func = thai2fit_tokenizer().word_tokenize + for rule in pre_rules: res = rule(res) res = tok_func(res) @@ -183,7 +186,7 @@ def document_vector(text: str, learn, data, agg: str = "mean"): """ - s = THAI2FIT_TOKENIZER.word_tokenize(text) + s = thai2fit_tokenizer().word_tokenize(text) t = torch.tensor(data.vocab.numericalize(s), requires_grad=False).to( device ) diff --git a/pythainlp/ulmfit/tokenizer.py b/pythainlp/ulmfit/tokenizer.py index a732891e7..0156a4ff8 100644 --- a/pythainlp/ulmfit/tokenizer.py +++ b/pythainlp/ulmfit/tokenizer.py @@ -8,7 +8,7 @@ from typing import Collection, List -from pythainlp.tokenize import THAI2FIT_TOKENIZER +from pythainlp.tokenize import thai2fit_tokenizer class BaseTokenizer: @@ -65,7 +65,7 @@ def tokenizer(text: str) -> List[str]: ' ', 'ภาวนามยปัญญา'] """ - return THAI2FIT_TOKENIZER.word_tokenize(text) + return thai2fit_tokenizer().word_tokenize(text) def add_special_cases(self, toks): pass diff --git a/pythainlp/util/phoneme.py b/pythainlp/util/phoneme.py index c473b2cd2..ecdb2d84f 100644 --- a/pythainlp/util/phoneme.py +++ b/pythainlp/util/phoneme.py @@ -5,6 +5,7 @@ """ Phonemes util """ +from functools import lru_cache import unicodedata from pythainlp.tokenize import Tokenizer @@ -192,8 +193,12 @@ def nectec_to_ipa(pronunciation: str) -> str: } dict_ipa_rtgs_final = {"w": "o"} -trie = Trie(list(dict_ipa_rtgs.keys()) + list(dict_ipa_rtgs_final.keys())) -ipa_cut = Tokenizer(custom_dict=trie, engine="newmm") + +@lru_cache +def _ipa_cut(): + """Lazy load IPA tokenizer with cache""" + trie = Trie(list(dict_ipa_rtgs.keys()) + list(dict_ipa_rtgs_final.keys())) + return Tokenizer(custom_dict=trie, engine="newmm") def ipa_to_rtgs(ipa: str) -> str: @@ -216,6 +221,7 @@ def ipa_to_rtgs(ipa: str) -> str: """ rtgs_parts = [] + ipa_cut = _ipa_cut() ipa_parts = ipa_cut.word_tokenize(ipa) for i, ipa_part in enumerate(ipa_parts): diff --git a/pythainlp/util/spell_words.py b/pythainlp/util/spell_words.py index 24b005e56..8ab43f0aa 100644 --- a/pythainlp/util/spell_words.py +++ b/pythainlp/util/spell_words.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from functools import lru_cache import re from typing import List @@ -48,7 +49,10 @@ for i in thai_below_vowels: dict_vowel[i] = "อ" + i -_cut = Tokenizer(list(dict_vowel.keys()) + list(thai_consonants), engine="mm") +@lru_cache +def _cut(): + """Lazy load vowel tokenizer with cache""" + return Tokenizer(list(dict_vowel.keys()) + list(thai_consonants), engine="mm") def _clean(w): @@ -93,7 +97,7 @@ def spell_syllable(text: str) -> List[str]: print(spell_syllable("แมว")) # output: ['มอ', 'วอ', 'แอ', 'แมว'] """ - tokens = _cut.word_tokenize(_clean(text)) + tokens = _cut().word_tokenize(_clean(text)) c_only = [tok + "อ" for tok in tokens if tok in set(thai_consonants)] v_only = [dict_vowel[tok] for tok in tokens if tok in set(dict_vowel)] diff --git a/pythainlp/util/time.py b/pythainlp/util/time.py index ff54cb91a..85738b94a 100644 --- a/pythainlp/util/time.py +++ b/pythainlp/util/time.py @@ -8,6 +8,7 @@ Convert time string or time object to Thai words. """ from datetime import datetime, time +from functools import lru_cache from typing import Union from pythainlp.tokenize import Tokenizer @@ -43,9 +44,13 @@ "นาฬิกา": 0, "ครึ่ง": 30, } -_THAI_TIME_CUT = Tokenizer( - custom_dict=list(_DICT_THAI_TIME.keys()), engine="newmm" -) + +@lru_cache +def _thai_time_cut(): + """Lazy load Thai time tokenizer with cache""" + return Tokenizer( + custom_dict=list(_DICT_THAI_TIME.keys()), engine="newmm" + ) _THAI_TIME_AFFIX = [ "โมงเช้า", "บ่ายโมง", @@ -266,10 +271,10 @@ def thaiword_to_time(text: str, padding: bool = True) -> str: _LIST_THAI_TIME = _time.split("|") del _time - hour = _THAI_TIME_CUT.word_tokenize(_LIST_THAI_TIME[0]) + hour = _thai_time_cut().word_tokenize(_LIST_THAI_TIME[0]) minute = _LIST_THAI_TIME[1] if len(minute) > 1: - minute = _THAI_TIME_CUT.word_tokenize(minute) + minute = _thai_time_cut().word_tokenize(minute) else: minute = 0 text = "" diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 2e08d3b6a..6b320b41e 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -8,6 +8,7 @@ First version of the code adapted from Korakot Chaovavanich's notebook https://colab.research.google.com/drive/148WNIeclf0kOU6QxKd6pcfwpSs8l-VKD#scrollTo=EuVDd0nNuI8Q """ +from functools import lru_cache import re from typing import List @@ -47,7 +48,10 @@ _valid_tokens = ( set(_digits.keys()) | set(_powers_of_10.keys()) | {"ล้าน", "ลบ"} ) -_tokenizer = Tokenizer(custom_dict=_valid_tokens) +@lru_cache +def _tokenizer(): + """Lazy load Thai numeral tokenizer with cache""" + return Tokenizer(custom_dict=_valid_tokens) def _check_is_thainum(word: str): @@ -60,11 +64,13 @@ def _check_is_thainum(word: str): return (False, None) -_dict_words = [i for i in list(thai_words()) if not _check_is_thainum(i)[0]] -_dict_words += list(_digits.keys()) -_dict_words += ["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"] - -_tokenizer_thaiwords = Tokenizer(_dict_words) +@lru_cache +def _tokenizer_thaiwords(): + """Lazy load Thai words tokenizer with cache""" + _dict_words = [i for i in list(thai_words()) if not _check_is_thainum(i)[0]] + _dict_words += list(_digits.keys()) + _dict_words += ["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"] + return Tokenizer(_dict_words) def thaiword_to_num(word: str) -> int: @@ -96,7 +102,7 @@ def thaiword_to_num(word: str) -> int: if not _re_thai_numerals.fullmatch(word): raise ValueError("The input string is not a valid Thai numeral") - tokens = _tokenizer.word_tokenize(word) + tokens = _tokenizer().word_tokenize(word) accumulated = 0 next_digit = 1 @@ -185,7 +191,7 @@ def text_to_num(text: str) -> List[str]: # output: ['10021889', 'บาท'] """ - _temp = _tokenizer_thaiwords.word_tokenize(text) + _temp = _tokenizer_thaiwords().word_tokenize(text) thainum = [] last_index = -1 list_word_new = [] diff --git a/pythainlp/word_vector/core.py b/pythainlp/word_vector/core.py index 777c38df2..886c596f2 100644 --- a/pythainlp/word_vector/core.py +++ b/pythainlp/word_vector/core.py @@ -9,7 +9,7 @@ from numpy import ndarray, zeros from pythainlp.corpus import get_corpus_path -from pythainlp.tokenize import THAI2FIT_TOKENIZER, word_tokenize +from pythainlp.tokenize import thai2fit_tokenizer, word_tokenize WV_DIM = 300 # word vector dimension @@ -61,7 +61,7 @@ def load_wordvector(self, model_name: str): self.WV_DIM = self.model.vector_size if self.model_name == "thai2fit_wv": - self.tokenize = THAI2FIT_TOKENIZER.word_tokenize + self.tokenize = thai2fit_tokenizer().word_tokenize else: self.tokenize = word_tokenize diff --git a/tests/core/test_tokenize.py b/tests/core/test_tokenize.py index bd5562eb3..e5672506d 100644 --- a/tests/core/test_tokenize.py +++ b/tests/core/test_tokenize.py @@ -6,7 +6,7 @@ import unittest from pythainlp.tokenize import ( - DEFAULT_WORD_DICT_TRIE, + word_dict_trie, Tokenizer, etcc, longest, @@ -261,7 +261,7 @@ def test_numeric_data_format(self): class TokenizeTestCase(unittest.TestCase): def test_Tokenizer(self): - _tokenizer = Tokenizer(DEFAULT_WORD_DICT_TRIE) + _tokenizer = Tokenizer(word_dict_trie()) self.assertEqual(_tokenizer.word_tokenize(""), []) _tokenizer.set_tokenize_engine("longest") self.assertEqual(_tokenizer.word_tokenize(None), []) diff --git a/tests/extra/testx_tokenize.py b/tests/extra/testx_tokenize.py index 321c7c747..e24e7eeb6 100644 --- a/tests/extra/testx_tokenize.py +++ b/tests/extra/testx_tokenize.py @@ -8,7 +8,7 @@ import unittest from pythainlp.tokenize import ( - DEFAULT_WORD_DICT_TRIE, + word_dict_trie, attacut, deepcut, nercut, @@ -275,12 +275,12 @@ def test_word_tokenize_deepcut(self): def test_deepcut(self): self.assertEqual(deepcut.segment(None), []) self.assertEqual(deepcut.segment(""), []) - self.assertIsNotNone(deepcut.segment("ทดสอบ", DEFAULT_WORD_DICT_TRIE)) + self.assertIsNotNone(deepcut.segment("ทดสอบ", word_dict_trie())) self.assertIsNotNone(deepcut.segment("ทดสอบ", ["ทด", "สอบ"])) self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut")) self.assertIsNotNone( word_tokenize( - "ทดสอบ", engine="deepcut", custom_dict=DEFAULT_WORD_DICT_TRIE + "ทดสอบ", engine="deepcut", custom_dict=word_dict_trie() ) )