Merge pull request #1186 from what-in-the-nim/reduce-peak-memory

Reduce peak memory on import by 62x

Author: bact

docs/api/spell.rst

@@ -48,9 +48,9 @@ The `NorvigSpellChecker` class is a fundamental component of the `pythainlp.spel
 DEFAULT_SPELL_CHECKER
 ~~~~~~~~~~~~~~~~~~~~~
 .. autodata:: DEFAULT_SPELL_CHECKER
-   :annotation: = Default instance of the standard NorvigSpellChecker, using word list data from the Thai National Corpus: http://www.arts.chula.ac.th/ling/tnc/
+   :annotation: = Default reference of the standard NorvigSpellChecker, using word list data from the Thai National Corpus: http://www.arts.chula.ac.th/ling/tnc/
 
-The `DEFAULT_SPELL_CHECKER` is an instance of the `NorvigSpellChecker` class with default settings. It is pre-configured to use word list data from the Thai National Corpus, making it a reliable choice for general spell-checking tasks.
+The `DEFAULT_SPELL_CHECKER` is an reference to the `NorvigSpellChecker` class with default settings. It is pre-configured to use word list data from the Thai National Corpus, making it a reliable choice for general spell-checking tasks.
 
 References
 ----------

pythainlp/augment/word2vec/thai2fit.py

@@ -6,7 +6,7 @@
 
 from pythainlp.augment.word2vec.core import Word2VecAug
 from pythainlp.corpus import get_corpus_path
-from pythainlp.tokenize import THAI2FIT_TOKENIZER
+from pythainlp.tokenize import thai2fit_tokenizer
 
 
 class Thai2fitAug:
@@ -26,7 +26,8 @@ def tokenizer(self, text: str) -> List[str]:
         :param str text: Thai text
         :rtype: List[str]
         """
-        return THAI2FIT_TOKENIZER.word_tokenize(text)
+        tok = thai2fit_tokenizer()
+        return tok.word_tokenize(text)
 
     def load_w2v(self):
         """

pythainlp/spell/__init__.py

@@ -18,7 +18,7 @@
 
 from pythainlp.spell.pn import NorvigSpellChecker
 
-DEFAULT_SPELL_CHECKER = NorvigSpellChecker()
+DEFAULT_SPELL_CHECKER = NorvigSpellChecker
 
 # these imports are placed here to avoid circular imports
 from pythainlp.spell.core import correct, correct_sent, spell, spell_sent

pythainlp/spell/core.py

@@ -6,11 +6,16 @@
 Spell checking functions
 """
 
+from functools import lru_cache
 import itertools
 from typing import List
 
 from pythainlp.spell import DEFAULT_SPELL_CHECKER
 
+@lru_cache
+def default_spell_checker():
+    """Lazy load default spell checker with cache"""
+    return DEFAULT_SPELL_CHECKER()
 
 def spell(word: str, engine: str = "pn") -> List[str]:
     """
@@ -72,7 +77,7 @@ def spell(word: str, engine: str = "pn") -> List[str]:
 
         text_correct = SPELL_CHECKER(word)
     else:
-        text_correct = DEFAULT_SPELL_CHECKER.spell(word)
+        text_correct = default_spell_checker().spell(word)
 
     return text_correct
 
@@ -125,7 +130,7 @@ def correct(word: str, engine: str = "pn") -> str:
         text_correct = SPELL_CHECKER(word)
 
     else:
-        text_correct = DEFAULT_SPELL_CHECKER.correct(word)
+        text_correct = default_spell_checker().correct(word)
 
     return text_correct
 

pythainlp/tokenize/__init__.py

@@ -7,7 +7,7 @@
 """
 
 __all__ = [
-    "THAI2FIT_TOKENIZER",
+    "thai2fit_tokenizer",
     "Tokenizer",
     "Trie",
     "paragraph_tokenize",
@@ -19,6 +19,7 @@
     "display_cell_tokenize",
 ]
 
+from functools import lru_cache
 from pythainlp.corpus import thai_syllables, thai_words
 from pythainlp.util.trie import Trie
 
@@ -27,9 +28,15 @@
 DEFAULT_SUBWORD_TOKENIZE_ENGINE = "tcc"
 DEFAULT_SYLLABLE_TOKENIZE_ENGINE = "han_solo"
 
-DEFAULT_WORD_DICT_TRIE = Trie(thai_words())
-DEFAULT_SYLLABLE_DICT_TRIE = Trie(thai_syllables())
-DEFAULT_DICT_TRIE = DEFAULT_WORD_DICT_TRIE
+@lru_cache
+def word_dict_trie():
+    """Lazy load default word dict trie with cache"""
+    return Trie(thai_words())
+
+@lru_cache
+def syllable_dict_trie():
+    """Lazy load default syllable dict trie with cache"""
+    return Trie(thai_syllables())
 
 from pythainlp.tokenize.core import (
     Tokenizer,
@@ -41,9 +48,4 @@
     word_tokenize,
     display_cell_tokenize,
 )
-
-from pythainlp.corpus import get_corpus as _get_corpus
-
-THAI2FIT_TOKENIZER = Tokenizer(
-    custom_dict=_get_corpus("words_th_thai2fit_201810.txt"), engine="mm"
-)
+from pythainlp.tokenize.thai2fit import thai2fit_tokenizer

pythainlp/tokenize/core.py

@@ -8,14 +8,14 @@
 
 import copy
 import re
-from typing import Iterable, List, Union
+from typing import Iterable, List, Optional, Union
 
 from pythainlp.tokenize import (
     DEFAULT_SENT_TOKENIZE_ENGINE,
     DEFAULT_SUBWORD_TOKENIZE_ENGINE,
-    DEFAULT_SYLLABLE_DICT_TRIE,
+    syllable_dict_trie,
     DEFAULT_SYLLABLE_TOKENIZE_ENGINE,
-    DEFAULT_WORD_DICT_TRIE,
+    word_dict_trie,
     DEFAULT_WORD_TOKENIZE_ENGINE,
 )
 from pythainlp.tokenize._utils import (
@@ -97,7 +97,7 @@ def word_detokenize(
 
 def word_tokenize(
     text: str,
-    custom_dict: Trie = Trie([]),
+    custom_dict: Optional[Trie] = None,
     engine: str = DEFAULT_WORD_TOKENIZE_ENGINE,
     keep_whitespace: bool = True,
     join_broken_num: bool = True,
@@ -223,6 +223,9 @@ def word_tokenize(
 
     segments = []
 
+    if custom_dict is None:
+        custom_dict = Trie([])
+
     if custom_dict and engine in (
         "attacut",
         "icu",
@@ -690,7 +693,7 @@ def subword_tokenize(
         for word in words:
             segments.extend(
                 word_tokenize(
-                    text=word, custom_dict=DEFAULT_SYLLABLE_DICT_TRIE
+                    text=word, custom_dict=syllable_dict_trie()
                 )
             )
     elif engine == "ssg":
@@ -881,7 +884,7 @@ def __init__(
         if custom_dict:
             self.__trie_dict = dict_trie(custom_dict)
         else:
-            self.__trie_dict = DEFAULT_WORD_DICT_TRIE
+            self.__trie_dict = word_dict_trie()
         self.__engine = engine
         if self.__engine not in ["newmm", "mm", "longest", "deepcut"]:
             raise NotImplementedError(

pythainlp/tokenize/etcc.py

@@ -19,14 +19,19 @@
 and backward longest matching techniques." In International Symposium on
 Communications and Information Technology (ISCIT), pp. 37-40. 2001.
 """
+from functools import lru_cache
 import re
 from typing import List
 
 from pythainlp import thai_follow_vowels
 from pythainlp.corpus import get_corpus
 from pythainlp.tokenize import Tokenizer
 
-_cut_etcc = Tokenizer(get_corpus("etcc.txt"), engine="longest")
+@lru_cache
+def _cut_etcc():
+    """Lazy load ETCC tokenizer with cache"""
+    return Tokenizer(get_corpus("etcc.txt"), engine="longest")
+
 _PAT_ENDING_CHAR = f"[{thai_follow_vowels}ๆฯ]"
 _RE_ENDING_CHAR = re.compile(_PAT_ENDING_CHAR)
 
@@ -64,4 +69,4 @@ def segment(text: str) -> List[str]:
     if not text or not isinstance(text, str):
         return []
 
-    return _cut_subword(_cut_etcc.word_tokenize(text))
+    return _cut_subword(_cut_etcc().word_tokenize(text))

pythainlp/tokenize/longest.py

@@ -12,10 +12,10 @@
 
 """
 import re
-from typing import Dict, List, Union
+from typing import Dict, List, Optional, Union
 
 from pythainlp import thai_tonemarks
-from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE
+from pythainlp.tokenize import word_dict_trie
 from pythainlp.util import Trie
 
 _FRONT_DEP_CHAR = [
@@ -152,7 +152,7 @@ def tokenize(self, text: str) -> List[str]:
 _tokenizers: Dict[int, LongestMatchTokenizer] = {}
 
 
-def segment(text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE) -> List[str]:
+def segment(text: str, custom_dict: Optional[Trie] = None) -> List[str]:
     """
     Dictionary-based longest matching word segmentation.
 
@@ -164,7 +164,7 @@ def segment(text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE) -> List[str]:
         return []
 
     if not custom_dict:
-        custom_dict = DEFAULT_WORD_DICT_TRIE
+        custom_dict = word_dict_trie()
 
     global _tokenizers
     custom_dict_ref_id = id(custom_dict)

pythainlp/tokenize/multi_cut.py

@@ -15,9 +15,9 @@
 
 import re
 from collections import defaultdict
-from typing import Iterator, List
+from typing import Iterator, List, Optional
 
-from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE
+from pythainlp.tokenize import word_dict_trie
 from pythainlp.util import Trie
 
 
@@ -48,12 +48,11 @@ def __init__(self, value, multi=None, in_dict=True):
 
 
 def _multicut(
-    text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE
+    text: str, custom_dict: Optional[Trie] = None
 ) -> Iterator[LatticeString]:
     """Return LatticeString"""
     if not custom_dict:
-        custom_dict = DEFAULT_WORD_DICT_TRIE
-
+        custom_dict = word_dict_trie()
     len_text = len(text)
     words_at = defaultdict(list)  # main data structure
 
@@ -123,40 +122,46 @@ def _combine(ww: List[LatticeString]) -> Iterator[str]:
 
 
 def segment(
-    text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE
+    text: str, custom_dict: Optional[Trie] = None
 ) -> List[str]:
     """Dictionary-based maximum matching word segmentation.
 
     :param text: text to be tokenized
     :type text: str
     :param custom_dict: tokenization dictionary,\
-        defaults to DEFAULT_WORD_DICT_TRIE
+        defaults to a Trie generated from pythainlp.corpus.thai_words
     :type custom_dict: Trie, optional
     :return: list of segmented tokens
     :rtype: List[str]
     """
     if not text or not isinstance(text, str):
         return []
 
+    if not custom_dict:
+        custom_dict = word_dict_trie()
+
     return list(_multicut(text, custom_dict=custom_dict))
 
 
 def find_all_segment(
-    text: str, custom_dict: Trie = DEFAULT_WORD_DICT_TRIE
+    text: str, custom_dict: Optional[Trie] = None
 ) -> List[str]:
     """Get all possible segment variations.
 
     :param text: input string to be tokenized
     :type text: str
     :param custom_dict: tokenization dictionary,\
-        defaults to DEFAULT_WORD_DICT_TRIE
+        defaults to word_dict_trie()
     :type custom_dict: Trie, optional
     :return: list of segment variations
     :rtype: List[str]
     """
     if not text or not isinstance(text, str):
         return []
 
+    if not custom_dict:
+        custom_dict = word_dict_trie()
+
     ww = list(_multicut(text, custom_dict=custom_dict))
 
     return list(_combine(ww))

pythainlp/tokenize/newmm.py

@@ -18,9 +18,9 @@
 import re
 from collections import defaultdict
 from heapq import heappop, heappush
-from typing import Generator, List
+from typing import Generator, List, Optional
 
-from pythainlp.tokenize import DEFAULT_WORD_DICT_TRIE
+from pythainlp.tokenize import word_dict_trie
 from pythainlp.tokenize.tcc_p import tcc_pos
 from pythainlp.util import Trie
 
@@ -140,7 +140,7 @@ def _onecut(text: str, custom_dict: Trie) -> Generator[str, None, None]:
 
 def segment(
     text: str,
-    custom_dict: Trie = DEFAULT_WORD_DICT_TRIE,
+    custom_dict: Optional[Trie] = None,
     safe_mode: bool = False,
 ) -> List[str]:
     """Maximal-matching word segmentation constrained by Thai Character Cluster.
@@ -153,7 +153,7 @@ def segment(
     :param text: text to be tokenized
     :type text: str
     :param custom_dict: tokenization dictionary,\
-        defaults to DEFAULT_WORD_DICT_TRIE
+        defaults to word_dict_trie()
     :type custom_dict: Trie, optional
     :param safe_mode: reduce chance for long processing time for long text\
         with many ambiguous breaking points, defaults to False
@@ -165,7 +165,7 @@ def segment(
         return []
 
     if not custom_dict:
-        custom_dict = DEFAULT_WORD_DICT_TRIE
+        custom_dict = word_dict_trie()
 
     if not safe_mode or len(text) < _TEXT_SCAN_END:
         return list(_onecut(text, custom_dict))