From de2a65c45e9fed5d97b21a0f015197941f3dc5c9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 16 Jan 2026 19:50:36 +0000 Subject: [PATCH 1/9] Initial plan From c659a3927a3c6e19b4a489e637c58e01108d6b9f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 16 Jan 2026 19:58:50 +0000 Subject: [PATCH 2/9] Make package zip-safe by using importlib.resources Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pyproject.toml | 1 - pythainlp/corpus/core.py | 27 +++++++++------- pythainlp/corpus/th_en_translit.py | 47 +++++++++++++++------------- pythainlp/spell/symspellpy.py | 50 ++++++++++++++++++++---------- pythainlp/tag/crfchunk.py | 10 ++++-- pythainlp/tokenize/han_solo.py | 19 ++++++++++-- pythainlp/tokenize/nlpo3.py | 24 +++++++++++--- pythainlp/tools/path.py | 12 +++++-- 8 files changed, 127 insertions(+), 63 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9a475166b..cf3c6f5a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -246,7 +246,6 @@ issues = "https://github.com/PyThaiNLP/pythainlp/issues" thainlp = "pythainlp.__main__:main" [tool.setuptools] -zip-safe = false include-package-data = true [tool.setuptools.packages.find] diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index e27af19f7..e54a24d6f 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -9,6 +9,7 @@ import json import os import re +from importlib.resources import files from pythainlp import __version__ from pythainlp.corpus import corpus_db_path, corpus_db_url, corpus_path @@ -126,10 +127,11 @@ def get_corpus(filename: str, comments: bool = True) -> frozenset: # ...}) """ - path = path_pythainlp_corpus(filename) - lines = [] - with open(path, encoding="utf-8-sig") as fh: - lines = fh.read().splitlines() + import pythainlp.corpus + corpus_files = files(pythainlp.corpus) + corpus_file = corpus_files.joinpath(filename) + text = corpus_file.read_text(encoding="utf-8-sig") + lines = text.splitlines() if not comments: # if the line has a '#' character, take only text before the first '#' @@ -165,10 +167,11 @@ def get_corpus_as_is(filename: str) -> list: # output: # ['แต่', 'ไม่'] """ - path = path_pythainlp_corpus(filename) - lines = [] - with open(path, encoding="utf-8-sig") as fh: - lines = fh.read().splitlines() + import pythainlp.corpus + corpus_files = files(pythainlp.corpus) + corpus_file = corpus_files.joinpath(filename) + text = corpus_file.read_text(encoding="utf-8-sig") + lines = text.splitlines() return lines @@ -184,9 +187,11 @@ def get_corpus_default_db(name: str, version: str = "") -> str | None: If you want to edit default_db.json, \ you can edit pythainlp/corpus/default_db.json """ - default_db_path = path_pythainlp_corpus("default_db.json") - with open(default_db_path, encoding="utf-8-sig") as fh: - corpus_db = json.load(fh) + import pythainlp.corpus + corpus_files = files(pythainlp.corpus) + default_db_file = corpus_files.joinpath("default_db.json") + text = default_db_file.read_text(encoding="utf-8-sig") + corpus_db = json.loads(text) if name in corpus_db: if version in corpus_db[name]["versions"]: diff --git a/pythainlp/corpus/th_en_translit.py b/pythainlp/corpus/th_en_translit.py index 94a73f922..e968d9000 100644 --- a/pythainlp/corpus/th_en_translit.py +++ b/pythainlp/corpus/th_en_translit.py @@ -17,8 +17,7 @@ ] from collections import defaultdict - -from pythainlp.corpus import path_pythainlp_corpus +from importlib.resources import files _FILE_NAME = "th_en_transliteration_v1.4.tsv" TRANSLITERATE_EN = "en" @@ -30,8 +29,11 @@ def get_transliteration_dict() -> defaultdict: The returned dict is in dict[str, dict[List[str], List[Optional[bool]]]] format. """ - path = path_pythainlp_corpus(_FILE_NAME) - if not path: + import pythainlp.corpus + corpus_files = files(pythainlp.corpus) + corpus_file = corpus_files.joinpath(_FILE_NAME) + + if not corpus_file.is_file(): raise FileNotFoundError( f"Unable to load transliteration dictionary. " f"{_FILE_NAME} is not found under pythainlp/corpus." @@ -42,24 +44,25 @@ def get_transliteration_dict() -> defaultdict: lambda: {TRANSLITERATE_EN: [], TRANSLITERATE_FOLLOW_RTSG: []} ) try: - with open(path, encoding="utf-8") as f: - # assume that the first row contains column names, so skip it. - for line in f.readlines()[1:]: - stripped = line.strip() - if stripped: - th, *en_checked = stripped.split("\t") - # replace in-between whitespace to prevent mismatched results from different tokenizers. - # e.g. "บอยแบนด์" - # route 1: "บอยแบนด์" -> ["บอย", "แบนด์"] -> ["boy", "band"] -> "boyband" - # route 2: "บอยแบนด์" -> [""บอยแบนด์""] -> ["boy band"] -> "boy band" - en_translit = en_checked[0].replace(" ", "") - trans_dict[th][TRANSLITERATE_EN].append(en_translit) - en_follow_rtgs = ( - bool(en_checked[1]) if len(en_checked) == 2 else None - ) - trans_dict[th][TRANSLITERATE_FOLLOW_RTSG].append( - en_follow_rtgs - ) + text = corpus_file.read_text(encoding="utf-8") + lines = text.splitlines() + # assume that the first row contains column names, so skip it. + for line in lines[1:]: + stripped = line.strip() + if stripped: + th, *en_checked = stripped.split("\t") + # replace in-between whitespace to prevent mismatched results from different tokenizers. + # e.g. "บอยแบนด์" + # route 1: "บอยแบนด์" -> ["บอย", "แบนด์"] -> ["boy", "band"] -> "boyband" + # route 2: "บอยแบนด์" -> [""บอยแบนด์""] -> ["boy band"] -> "boy band" + en_translit = en_checked[0].replace(" ", "") + trans_dict[th][TRANSLITERATE_EN].append(en_translit) + en_follow_rtgs = ( + bool(en_checked[1]) if len(en_checked) == 2 else None + ) + trans_dict[th][TRANSLITERATE_FOLLOW_RTSG].append( + en_follow_rtgs + ) except ValueError as exc: raise ValueError( diff --git a/pythainlp/spell/symspellpy.py b/pythainlp/spell/symspellpy.py index 8ba10fa22..6903e9fbc 100644 --- a/pythainlp/spell/symspellpy.py +++ b/pythainlp/spell/symspellpy.py @@ -13,6 +13,8 @@ from __future__ import annotations +from importlib.resources import as_file, files + try: from symspellpy import SymSpell, Verbosity except ImportError: @@ -20,29 +22,44 @@ "Import Error; Install symspellpy by pip install symspellpy" ) -from pythainlp.corpus import get_corpus_path, path_pythainlp_corpus +from pythainlp.corpus import get_corpus_path _UNIGRAM_FILENAME = "tnc_freq.txt" _BIGRAM_CORPUS_NAME = "tnc_bigram_word_freqs" -sym_spell = SymSpell() -sym_spell.load_dictionary( - path_pythainlp_corpus(_UNIGRAM_FILENAME), - 0, - 1, - separator="\t", - encoding="utf-8-sig", -) -sym_spell.load_bigram_dictionary( - get_corpus_path(_BIGRAM_CORPUS_NAME), - 0, - 2, - separator="\t", - encoding="utf-8-sig", -) +_sym_spell = None + + +def _get_sym_spell(): + """Lazy load the symspell instance.""" + global _sym_spell + if _sym_spell is None: + _sym_spell = SymSpell() + # Load unigram dictionary from bundled corpus + import pythainlp.corpus + corpus_files = files(pythainlp.corpus) + unigram_file = corpus_files.joinpath(_UNIGRAM_FILENAME) + with as_file(unigram_file) as unigram_path: + _sym_spell.load_dictionary( + str(unigram_path), + 0, + 1, + separator="\t", + encoding="utf-8-sig", + ) + # Load bigram dictionary from downloaded corpus + _sym_spell.load_bigram_dictionary( + get_corpus_path(_BIGRAM_CORPUS_NAME), + 0, + 2, + separator="\t", + encoding="utf-8-sig", + ) + return _sym_spell def spell(text: str, max_edit_distance: int = 2) -> list[str]: + sym_spell = _get_sym_spell() return [ str(i).split(",", maxsplit=1)[0] for i in list( @@ -60,6 +77,7 @@ def correct(text: str, max_edit_distance: int = 1) -> str: def spell_sent( list_words: list[str], max_edit_distance: int = 2 ) -> list[list[str]]: + sym_spell = _get_sym_spell() temp = [ str(i).split(",", maxsplit=1)[0].split(" ") for i in list( diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py index 1e949a817..1717b2bf7 100644 --- a/pythainlp/tag/crfchunk.py +++ b/pythainlp/tag/crfchunk.py @@ -3,9 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from importlib.resources import as_file, files from pycrfsuite import Tagger as CRFTagger -from pythainlp.corpus import path_pythainlp_corpus, thai_stopwords +from pythainlp.corpus import thai_stopwords def _is_stopword(word: str) -> bool: # check Thai stopword @@ -62,8 +63,11 @@ def __init__(self, corpus: str = "orchidpp"): def load_model(self, corpus: str): self.tagger = CRFTagger() if corpus == "orchidpp": - self.path = path_pythainlp_corpus("crfchunk_orchidpp.model") - self.tagger.open(self.path) + import pythainlp.corpus + corpus_files = files(pythainlp.corpus) + model_file = corpus_files.joinpath("crfchunk_orchidpp.model") + with as_file(model_file) as model_path: + self.tagger.open(str(model_path)) def parse(self, token_pos: list[tuple[str, str]]) -> list[str]: self.xseq = extract_features(token_pos) diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py index 4c11ce684..efea6c0c8 100644 --- a/pythainlp/tokenize/han_solo.py +++ b/pythainlp/tokenize/han_solo.py @@ -8,7 +8,7 @@ from __future__ import annotations -from pythainlp.corpus import path_pythainlp_corpus +from importlib.resources import as_file, files try: import pycrfsuite @@ -17,8 +17,20 @@ "ImportError; Install pycrfsuite by pip install python-crfsuite" ) -tagger = pycrfsuite.Tagger() -tagger.open(path_pythainlp_corpus("han_solo.crfsuite")) +_tagger = None + + +def _get_tagger(): + """Lazy load the tagger model.""" + global _tagger + if _tagger is None: + _tagger = pycrfsuite.Tagger() + import pythainlp.corpus + corpus_files = files(pythainlp.corpus) + model_file = corpus_files.joinpath("han_solo.crfsuite") + with as_file(model_file) as model_path: + _tagger.open(str(model_path)) + return _tagger class Featurizer: @@ -119,6 +131,7 @@ def featurize( def segment(text: str) -> list[str]: + tagger = _get_tagger() x = _to_feature.featurize(text)["X"] y_pred = tagger.tag(x) list_cut = [] diff --git a/pythainlp/tokenize/nlpo3.py b/pythainlp/tokenize/nlpo3.py index 310928ee1..8f3e631fb 100644 --- a/pythainlp/tokenize/nlpo3.py +++ b/pythainlp/tokenize/nlpo3.py @@ -3,18 +3,30 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from importlib.resources import as_file, files from sys import stderr from nlpo3 import load_dict as nlpo3_load_dict from nlpo3 import segment as nlpo3_segment -from pythainlp.corpus import path_pythainlp_corpus from pythainlp.corpus.common import _THAI_WORDS_FILENAME _NLPO3_DEFAULT_DICT_NAME = "_73bcj049dzbu9t49b4va170k" # supposed to be unique -_NLPO3_DEFAULT_DICT = nlpo3_load_dict( - path_pythainlp_corpus(_THAI_WORDS_FILENAME), _NLPO3_DEFAULT_DICT_NAME -) # preload default dict, so it can be accessible by _NLPO3_DEFAULT_DICT_NAME +_NLPO3_DEFAULT_DICT = None # Will be lazily loaded + + +def _ensure_default_dict_loaded(): + """Ensure the default dictionary is loaded.""" + global _NLPO3_DEFAULT_DICT + if _NLPO3_DEFAULT_DICT is None: + import pythainlp.corpus + corpus_files = files(pythainlp.corpus) + dict_file = corpus_files.joinpath(_THAI_WORDS_FILENAME) + with as_file(dict_file) as dict_path: + _NLPO3_DEFAULT_DICT = nlpo3_load_dict( + str(dict_path), _NLPO3_DEFAULT_DICT_NAME + ) + return _NLPO3_DEFAULT_DICT def load_dict(file_path: str, dict_name: str) -> bool: @@ -64,6 +76,10 @@ def segment( * \ https://github.com/PyThaiNLP/nlpo3 """ + # Ensure default dict is loaded if it's being used + if custom_dict == _NLPO3_DEFAULT_DICT_NAME: + _ensure_default_dict_loaded() + return nlpo3_segment( text=text, dict_name=custom_dict, diff --git a/pythainlp/tools/path.py b/pythainlp/tools/path.py index 9af51e60a..e12b497bc 100644 --- a/pythainlp/tools/path.py +++ b/pythainlp/tools/path.py @@ -9,8 +9,7 @@ from __future__ import annotations import os - -from pythainlp import __file__ as pythainlp_file +from importlib.resources import files PYTHAINLP_DEFAULT_DATA_DIR = "pythainlp-data" @@ -73,4 +72,11 @@ def get_pythainlp_path() -> str: get_pythainlp_path() # output: '/usr/local/lib/python3.6/dist-packages/pythainlp' """ - return os.path.dirname(pythainlp_file) + import pythainlp + package_path = files(pythainlp) + # For compatibility, convert to string path if possible + # This works for both regular installations and zip files + if hasattr(package_path, '__fspath__'): + return os.fspath(package_path) + # Fallback for older Python or special cases + return str(package_path) From 51dfad3cd0ebc0b111b4232082bb1b57fc6aeabe Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 16 Jan 2026 20:01:10 +0000 Subject: [PATCH 3/9] Fix import order in crfchunk.py Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/tag/crfchunk.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py index 1717b2bf7..71eb12d3a 100644 --- a/pythainlp/tag/crfchunk.py +++ b/pythainlp/tag/crfchunk.py @@ -4,6 +4,7 @@ from __future__ import annotations from importlib.resources import as_file, files + from pycrfsuite import Tagger as CRFTagger from pythainlp.corpus import thai_stopwords From 58a54c8dcf2b9b0fdd785c516161bd751b538b38 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 16 Jan 2026 20:02:47 +0000 Subject: [PATCH 4/9] Clarify comment in get_pythainlp_path fallback logic Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/tools/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythainlp/tools/path.py b/pythainlp/tools/path.py index e12b497bc..9cc2b6204 100644 --- a/pythainlp/tools/path.py +++ b/pythainlp/tools/path.py @@ -78,5 +78,5 @@ def get_pythainlp_path() -> str: # This works for both regular installations and zip files if hasattr(package_path, '__fspath__'): return os.fspath(package_path) - # Fallback for older Python or special cases + # Fallback for traversable objects that don't support __fspath__ return str(package_path) From c497e2ac4f89e49da1f0a6f40a9e0a5c696273b8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 16 Jan 2026 20:20:09 +0000 Subject: [PATCH 5/9] Keep as_file() context managers alive for model lifetime Address code review feedback: - Keep context managers open for pycrfsuite and SymSpell models - Use string imports for files() instead of module imports - Add proper cleanup in __del__ methods where appropriate Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/corpus/core.py | 9 +++------ pythainlp/corpus/th_en_translit.py | 3 +-- pythainlp/spell/symspellpy.py | 23 ++++++++++++----------- pythainlp/tag/crfchunk.py | 18 ++++++++++++++---- pythainlp/tokenize/han_solo.py | 11 ++++++----- pythainlp/tokenize/nlpo3.py | 15 ++++++++------- 6 files changed, 44 insertions(+), 35 deletions(-) diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index e54a24d6f..af4aca00a 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -127,8 +127,7 @@ def get_corpus(filename: str, comments: bool = True) -> frozenset: # ...}) """ - import pythainlp.corpus - corpus_files = files(pythainlp.corpus) + corpus_files = files("pythainlp.corpus") corpus_file = corpus_files.joinpath(filename) text = corpus_file.read_text(encoding="utf-8-sig") lines = text.splitlines() @@ -167,8 +166,7 @@ def get_corpus_as_is(filename: str) -> list: # output: # ['แต่', 'ไม่'] """ - import pythainlp.corpus - corpus_files = files(pythainlp.corpus) + corpus_files = files("pythainlp.corpus") corpus_file = corpus_files.joinpath(filename) text = corpus_file.read_text(encoding="utf-8-sig") lines = text.splitlines() @@ -187,8 +185,7 @@ def get_corpus_default_db(name: str, version: str = "") -> str | None: If you want to edit default_db.json, \ you can edit pythainlp/corpus/default_db.json """ - import pythainlp.corpus - corpus_files = files(pythainlp.corpus) + corpus_files = files("pythainlp.corpus") default_db_file = corpus_files.joinpath("default_db.json") text = default_db_file.read_text(encoding="utf-8-sig") corpus_db = json.loads(text) diff --git a/pythainlp/corpus/th_en_translit.py b/pythainlp/corpus/th_en_translit.py index e968d9000..57796de96 100644 --- a/pythainlp/corpus/th_en_translit.py +++ b/pythainlp/corpus/th_en_translit.py @@ -29,8 +29,7 @@ def get_transliteration_dict() -> defaultdict: The returned dict is in dict[str, dict[List[str], List[Optional[bool]]]] format. """ - import pythainlp.corpus - corpus_files = files(pythainlp.corpus) + corpus_files = files("pythainlp.corpus") corpus_file = corpus_files.joinpath(_FILE_NAME) if not corpus_file.is_file(): diff --git a/pythainlp/spell/symspellpy.py b/pythainlp/spell/symspellpy.py index 6903e9fbc..7cc1ba2e8 100644 --- a/pythainlp/spell/symspellpy.py +++ b/pythainlp/spell/symspellpy.py @@ -28,25 +28,26 @@ _BIGRAM_CORPUS_NAME = "tnc_bigram_word_freqs" _sym_spell = None +_unigram_path_ctx = None def _get_sym_spell(): """Lazy load the symspell instance.""" - global _sym_spell + global _sym_spell, _unigram_path_ctx if _sym_spell is None: _sym_spell = SymSpell() # Load unigram dictionary from bundled corpus - import pythainlp.corpus - corpus_files = files(pythainlp.corpus) + corpus_files = files("pythainlp.corpus") unigram_file = corpus_files.joinpath(_UNIGRAM_FILENAME) - with as_file(unigram_file) as unigram_path: - _sym_spell.load_dictionary( - str(unigram_path), - 0, - 1, - separator="\t", - encoding="utf-8-sig", - ) + _unigram_path_ctx = as_file(unigram_file) + unigram_path = _unigram_path_ctx.__enter__() + _sym_spell.load_dictionary( + str(unigram_path), + 0, + 1, + separator="\t", + encoding="utf-8-sig", + ) # Load bigram dictionary from downloaded corpus _sym_spell.load_bigram_dictionary( get_corpus_path(_BIGRAM_CORPUS_NAME), diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py index 71eb12d3a..8a538027a 100644 --- a/pythainlp/tag/crfchunk.py +++ b/pythainlp/tag/crfchunk.py @@ -59,17 +59,27 @@ def extract_features(doc): class CRFchunk: def __init__(self, corpus: str = "orchidpp"): self.corpus = corpus + self.model_path_ctx = None self.load_model(self.corpus) def load_model(self, corpus: str): self.tagger = CRFTagger() if corpus == "orchidpp": - import pythainlp.corpus - corpus_files = files(pythainlp.corpus) + corpus_files = files("pythainlp.corpus") model_file = corpus_files.joinpath("crfchunk_orchidpp.model") - with as_file(model_file) as model_path: - self.tagger.open(str(model_path)) + self.model_path_ctx = as_file(model_file) + model_path = self.model_path_ctx.__enter__() + self.tagger.open(str(model_path)) def parse(self, token_pos: list[tuple[str, str]]) -> list[str]: self.xseq = extract_features(token_pos) return self.tagger.tag(self.xseq) + + def __del__(self): + """Clean up the context manager when object is destroyed.""" + if self.model_path_ctx is not None: + try: + self.model_path_ctx.__exit__(None, None, None) + except Exception: # noqa: S110 + # Silently ignore cleanup errors during garbage collection + pass diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py index efea6c0c8..82954a3fd 100644 --- a/pythainlp/tokenize/han_solo.py +++ b/pythainlp/tokenize/han_solo.py @@ -18,18 +18,19 @@ ) _tagger = None +_model_path_ctx = None def _get_tagger(): """Lazy load the tagger model.""" - global _tagger + global _tagger, _model_path_ctx if _tagger is None: _tagger = pycrfsuite.Tagger() - import pythainlp.corpus - corpus_files = files(pythainlp.corpus) + corpus_files = files("pythainlp.corpus") model_file = corpus_files.joinpath("han_solo.crfsuite") - with as_file(model_file) as model_path: - _tagger.open(str(model_path)) + _model_path_ctx = as_file(model_file) + model_path = _model_path_ctx.__enter__() + _tagger.open(str(model_path)) return _tagger diff --git a/pythainlp/tokenize/nlpo3.py b/pythainlp/tokenize/nlpo3.py index 8f3e631fb..4c11070dd 100644 --- a/pythainlp/tokenize/nlpo3.py +++ b/pythainlp/tokenize/nlpo3.py @@ -13,19 +13,20 @@ _NLPO3_DEFAULT_DICT_NAME = "_73bcj049dzbu9t49b4va170k" # supposed to be unique _NLPO3_DEFAULT_DICT = None # Will be lazily loaded +_dict_path_ctx = None def _ensure_default_dict_loaded(): """Ensure the default dictionary is loaded.""" - global _NLPO3_DEFAULT_DICT + global _NLPO3_DEFAULT_DICT, _dict_path_ctx if _NLPO3_DEFAULT_DICT is None: - import pythainlp.corpus - corpus_files = files(pythainlp.corpus) + corpus_files = files("pythainlp.corpus") dict_file = corpus_files.joinpath(_THAI_WORDS_FILENAME) - with as_file(dict_file) as dict_path: - _NLPO3_DEFAULT_DICT = nlpo3_load_dict( - str(dict_path), _NLPO3_DEFAULT_DICT_NAME - ) + _dict_path_ctx = as_file(dict_file) + dict_path = _dict_path_ctx.__enter__() + _NLPO3_DEFAULT_DICT = nlpo3_load_dict( + str(dict_path), _NLPO3_DEFAULT_DICT_NAME + ) return _NLPO3_DEFAULT_DICT From df59213de9c1f7c9d4b952270223abbb7da1e5e5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 16 Jan 2026 21:18:32 +0000 Subject: [PATCH 6/9] Make context manager attribute private in CRFchunk Change self.model_path_ctx to self._model_path_ctx following Python convention for private attributes. The context manager is an implementation detail that users don't need to access. Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/tag/crfchunk.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py index 8a538027a..a0cc38190 100644 --- a/pythainlp/tag/crfchunk.py +++ b/pythainlp/tag/crfchunk.py @@ -59,7 +59,7 @@ def extract_features(doc): class CRFchunk: def __init__(self, corpus: str = "orchidpp"): self.corpus = corpus - self.model_path_ctx = None + self._model_path_ctx = None self.load_model(self.corpus) def load_model(self, corpus: str): @@ -67,8 +67,8 @@ def load_model(self, corpus: str): if corpus == "orchidpp": corpus_files = files("pythainlp.corpus") model_file = corpus_files.joinpath("crfchunk_orchidpp.model") - self.model_path_ctx = as_file(model_file) - model_path = self.model_path_ctx.__enter__() + self._model_path_ctx = as_file(model_file) + model_path = self._model_path_ctx.__enter__() self.tagger.open(str(model_path)) def parse(self, token_pos: list[tuple[str, str]]) -> list[str]: @@ -77,9 +77,9 @@ def parse(self, token_pos: list[tuple[str, str]]) -> list[str]: def __del__(self): """Clean up the context manager when object is destroyed.""" - if self.model_path_ctx is not None: + if self._model_path_ctx is not None: try: - self.model_path_ctx.__exit__(None, None, None) + self._model_path_ctx.__exit__(None, None, None) except Exception: # noqa: S110 # Silently ignore cleanup errors during garbage collection pass From be45633685da24885e40020e6d234b83fab188ca Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 16 Jan 2026 21:38:36 +0000 Subject: [PATCH 7/9] Add thread safety and improve resource management Address code review feedback: - Add threading locks to ensure thread-safe lazy loading - Implement double-check locking pattern to prevent race conditions - Make CRFchunk a context manager for explicit resource management - Document that __del__ is unreliable and context manager is preferred - Add documentation about zip-safe behavior in get_pythainlp_path() - Document that context managers are kept alive for program lifetime Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/spell/symspellpy.py | 56 ++++++++++++++++++++-------------- pythainlp/tag/crfchunk.py | 33 +++++++++++++++++++- pythainlp/tokenize/han_solo.py | 26 +++++++++++----- pythainlp/tokenize/nlpo3.py | 28 +++++++++++------ pythainlp/tools/path.py | 7 ++++- 5 files changed, 108 insertions(+), 42 deletions(-) diff --git a/pythainlp/spell/symspellpy.py b/pythainlp/spell/symspellpy.py index 7cc1ba2e8..2e10af8ea 100644 --- a/pythainlp/spell/symspellpy.py +++ b/pythainlp/spell/symspellpy.py @@ -13,6 +13,7 @@ from __future__ import annotations +import threading from importlib.resources import as_file, files try: @@ -28,34 +29,43 @@ _BIGRAM_CORPUS_NAME = "tnc_bigram_word_freqs" _sym_spell = None -_unigram_path_ctx = None +_unigram_path_ctx = None # Context manager kept alive for program lifetime +_load_lock = threading.Lock() # Thread safety for lazy loading def _get_sym_spell(): - """Lazy load the symspell instance.""" + """Lazy load the symspell instance. + + This function uses a lock to ensure thread-safe initialization. + The context manager is kept alive for the lifetime of the program + to prevent cleanup of temporary files while SymSpell is in use. + """ global _sym_spell, _unigram_path_ctx if _sym_spell is None: - _sym_spell = SymSpell() - # Load unigram dictionary from bundled corpus - corpus_files = files("pythainlp.corpus") - unigram_file = corpus_files.joinpath(_UNIGRAM_FILENAME) - _unigram_path_ctx = as_file(unigram_file) - unigram_path = _unigram_path_ctx.__enter__() - _sym_spell.load_dictionary( - str(unigram_path), - 0, - 1, - separator="\t", - encoding="utf-8-sig", - ) - # Load bigram dictionary from downloaded corpus - _sym_spell.load_bigram_dictionary( - get_corpus_path(_BIGRAM_CORPUS_NAME), - 0, - 2, - separator="\t", - encoding="utf-8-sig", - ) + with _load_lock: + # Double-check pattern to avoid race conditions + if _sym_spell is None: + _sym_spell = SymSpell() + # Load unigram dictionary from bundled corpus + corpus_files = files("pythainlp.corpus") + unigram_file = corpus_files.joinpath(_UNIGRAM_FILENAME) + _unigram_path_ctx = as_file(unigram_file) + unigram_path = _unigram_path_ctx.__enter__() + _sym_spell.load_dictionary( + str(unigram_path), + 0, + 1, + separator="\t", + encoding="utf-8-sig", + ) + # Load bigram dictionary from downloaded corpus + _sym_spell.load_bigram_dictionary( + get_corpus_path(_BIGRAM_CORPUS_NAME), + 0, + 2, + separator="\t", + encoding="utf-8-sig", + ) return _sym_spell diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py index a0cc38190..3ac7d292e 100644 --- a/pythainlp/tag/crfchunk.py +++ b/pythainlp/tag/crfchunk.py @@ -57,6 +57,18 @@ def extract_features(doc): class CRFchunk: + """CRF-based chunker for Thai text. + + This class can be used as a context manager to ensure proper cleanup + of resources. Example: + + with CRFchunk() as chunker: + result = chunker.parse(tokens) + + Alternatively, the object will attempt to clean up resources when + garbage collected, though this is not guaranteed. + """ + def __init__(self, corpus: str = "orchidpp"): self.corpus = corpus self._model_path_ctx = None @@ -75,8 +87,27 @@ def parse(self, token_pos: list[tuple[str, str]]) -> list[str]: self.xseq = extract_features(token_pos) return self.tagger.tag(self.xseq) + def __enter__(self): + """Context manager entry.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit - clean up resources.""" + if self._model_path_ctx is not None: + try: + self._model_path_ctx.__exit__(exc_type, exc_val, exc_tb) + self._model_path_ctx = None + except Exception: # noqa: S110 + pass + return False + def __del__(self): - """Clean up the context manager when object is destroyed.""" + """Clean up the context manager when object is destroyed. + + Note: __del__ is not guaranteed to be called and should not be + relied upon for critical cleanup. Use the context manager protocol + (with statement) for reliable resource management. + """ if self._model_path_ctx is not None: try: self._model_path_ctx.__exit__(None, None, None) diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py index 82954a3fd..3f5498647 100644 --- a/pythainlp/tokenize/han_solo.py +++ b/pythainlp/tokenize/han_solo.py @@ -8,6 +8,7 @@ from __future__ import annotations +import threading from importlib.resources import as_file, files try: @@ -18,19 +19,28 @@ ) _tagger = None -_model_path_ctx = None +_model_path_ctx = None # Context manager kept alive for program lifetime +_load_lock = threading.Lock() # Thread safety for lazy loading def _get_tagger(): - """Lazy load the tagger model.""" + """Lazy load the tagger model. + + This function uses a lock to ensure thread-safe initialization. + The context manager is kept alive for the lifetime of the program + to prevent cleanup of temporary files while the tagger is in use. + """ global _tagger, _model_path_ctx if _tagger is None: - _tagger = pycrfsuite.Tagger() - corpus_files = files("pythainlp.corpus") - model_file = corpus_files.joinpath("han_solo.crfsuite") - _model_path_ctx = as_file(model_file) - model_path = _model_path_ctx.__enter__() - _tagger.open(str(model_path)) + with _load_lock: + # Double-check pattern to avoid race conditions + if _tagger is None: + _tagger = pycrfsuite.Tagger() + corpus_files = files("pythainlp.corpus") + model_file = corpus_files.joinpath("han_solo.crfsuite") + _model_path_ctx = as_file(model_file) + model_path = _model_path_ctx.__enter__() + _tagger.open(str(model_path)) return _tagger diff --git a/pythainlp/tokenize/nlpo3.py b/pythainlp/tokenize/nlpo3.py index 4c11070dd..65734ea01 100644 --- a/pythainlp/tokenize/nlpo3.py +++ b/pythainlp/tokenize/nlpo3.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +import threading from importlib.resources import as_file, files from sys import stderr @@ -13,20 +14,29 @@ _NLPO3_DEFAULT_DICT_NAME = "_73bcj049dzbu9t49b4va170k" # supposed to be unique _NLPO3_DEFAULT_DICT = None # Will be lazily loaded -_dict_path_ctx = None +_dict_path_ctx = None # Context manager kept alive for program lifetime +_load_lock = threading.Lock() # Thread safety for lazy loading def _ensure_default_dict_loaded(): - """Ensure the default dictionary is loaded.""" + """Ensure the default dictionary is loaded. + + This function uses a lock to ensure thread-safe initialization. + The context manager is kept alive for the lifetime of the program + to prevent cleanup of temporary files while the dictionary is in use. + """ global _NLPO3_DEFAULT_DICT, _dict_path_ctx if _NLPO3_DEFAULT_DICT is None: - corpus_files = files("pythainlp.corpus") - dict_file = corpus_files.joinpath(_THAI_WORDS_FILENAME) - _dict_path_ctx = as_file(dict_file) - dict_path = _dict_path_ctx.__enter__() - _NLPO3_DEFAULT_DICT = nlpo3_load_dict( - str(dict_path), _NLPO3_DEFAULT_DICT_NAME - ) + with _load_lock: + # Double-check pattern to avoid race conditions + if _NLPO3_DEFAULT_DICT is None: + corpus_files = files("pythainlp.corpus") + dict_file = corpus_files.joinpath(_THAI_WORDS_FILENAME) + _dict_path_ctx = as_file(dict_file) + dict_path = _dict_path_ctx.__enter__() + _NLPO3_DEFAULT_DICT = nlpo3_load_dict( + str(dict_path), _NLPO3_DEFAULT_DICT_NAME + ) return _NLPO3_DEFAULT_DICT diff --git a/pythainlp/tools/path.py b/pythainlp/tools/path.py index 9cc2b6204..74e7cdcf8 100644 --- a/pythainlp/tools/path.py +++ b/pythainlp/tools/path.py @@ -59,7 +59,12 @@ def get_pythainlp_data_path() -> str: def get_pythainlp_path() -> str: - """This function returns full path of PyThaiNLP codes + """This function returns full path of PyThaiNLP codes. + + Note: When the package is installed as a zip file, the returned path + may not be a standard filesystem path and should not be used for direct + file I/O operations. Use importlib.resources for accessing package files + in a zip-safe manner. :return: full path of :mod:`pythainlp` codes :rtype: str From a9750f1106e4008f6dbe9eb2165c0ab886529b71 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 17 Jan 2026 10:12:41 +0000 Subject: [PATCH 8/9] Rename context manager variables for clarity Change variable names from *_path_ctx to *_file_ctx to better reflect that they are file context managers from as_file(), not path contexts: - _dict_path_ctx -> _dict_file_ctx - _model_path_ctx -> _model_file_ctx - _unigram_path_ctx -> _unigram_file_ctx - self._model_path_ctx -> self._model_file_ctx Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/spell/symspellpy.py | 8 ++++---- pythainlp/tag/crfchunk.py | 16 ++++++++-------- pythainlp/tokenize/han_solo.py | 8 ++++---- pythainlp/tokenize/nlpo3.py | 8 ++++---- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pythainlp/spell/symspellpy.py b/pythainlp/spell/symspellpy.py index 2e10af8ea..32919b75a 100644 --- a/pythainlp/spell/symspellpy.py +++ b/pythainlp/spell/symspellpy.py @@ -29,7 +29,7 @@ _BIGRAM_CORPUS_NAME = "tnc_bigram_word_freqs" _sym_spell = None -_unigram_path_ctx = None # Context manager kept alive for program lifetime +_unigram_file_ctx = None # File context manager kept alive for program lifetime _load_lock = threading.Lock() # Thread safety for lazy loading @@ -40,7 +40,7 @@ def _get_sym_spell(): The context manager is kept alive for the lifetime of the program to prevent cleanup of temporary files while SymSpell is in use. """ - global _sym_spell, _unigram_path_ctx + global _sym_spell, _unigram_file_ctx if _sym_spell is None: with _load_lock: # Double-check pattern to avoid race conditions @@ -49,8 +49,8 @@ def _get_sym_spell(): # Load unigram dictionary from bundled corpus corpus_files = files("pythainlp.corpus") unigram_file = corpus_files.joinpath(_UNIGRAM_FILENAME) - _unigram_path_ctx = as_file(unigram_file) - unigram_path = _unigram_path_ctx.__enter__() + _unigram_file_ctx = as_file(unigram_file) + unigram_path = _unigram_file_ctx.__enter__() _sym_spell.load_dictionary( str(unigram_path), 0, diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py index 3ac7d292e..332e2b33b 100644 --- a/pythainlp/tag/crfchunk.py +++ b/pythainlp/tag/crfchunk.py @@ -71,7 +71,7 @@ class CRFchunk: def __init__(self, corpus: str = "orchidpp"): self.corpus = corpus - self._model_path_ctx = None + self._model_file_ctx = None self.load_model(self.corpus) def load_model(self, corpus: str): @@ -79,8 +79,8 @@ def load_model(self, corpus: str): if corpus == "orchidpp": corpus_files = files("pythainlp.corpus") model_file = corpus_files.joinpath("crfchunk_orchidpp.model") - self._model_path_ctx = as_file(model_file) - model_path = self._model_path_ctx.__enter__() + self._model_file_ctx = as_file(model_file) + model_path = self._model_file_ctx.__enter__() self.tagger.open(str(model_path)) def parse(self, token_pos: list[tuple[str, str]]) -> list[str]: @@ -93,10 +93,10 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): """Context manager exit - clean up resources.""" - if self._model_path_ctx is not None: + if self._model_file_ctx is not None: try: - self._model_path_ctx.__exit__(exc_type, exc_val, exc_tb) - self._model_path_ctx = None + self._model_file_ctx.__exit__(exc_type, exc_val, exc_tb) + self._model_file_ctx = None except Exception: # noqa: S110 pass return False @@ -108,9 +108,9 @@ def __del__(self): relied upon for critical cleanup. Use the context manager protocol (with statement) for reliable resource management. """ - if self._model_path_ctx is not None: + if self._model_file_ctx is not None: try: - self._model_path_ctx.__exit__(None, None, None) + self._model_file_ctx.__exit__(None, None, None) except Exception: # noqa: S110 # Silently ignore cleanup errors during garbage collection pass diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py index 3f5498647..dc438f68a 100644 --- a/pythainlp/tokenize/han_solo.py +++ b/pythainlp/tokenize/han_solo.py @@ -19,7 +19,7 @@ ) _tagger = None -_model_path_ctx = None # Context manager kept alive for program lifetime +_model_file_ctx = None # File context manager kept alive for program lifetime _load_lock = threading.Lock() # Thread safety for lazy loading @@ -30,7 +30,7 @@ def _get_tagger(): The context manager is kept alive for the lifetime of the program to prevent cleanup of temporary files while the tagger is in use. """ - global _tagger, _model_path_ctx + global _tagger, _model_file_ctx if _tagger is None: with _load_lock: # Double-check pattern to avoid race conditions @@ -38,8 +38,8 @@ def _get_tagger(): _tagger = pycrfsuite.Tagger() corpus_files = files("pythainlp.corpus") model_file = corpus_files.joinpath("han_solo.crfsuite") - _model_path_ctx = as_file(model_file) - model_path = _model_path_ctx.__enter__() + _model_file_ctx = as_file(model_file) + model_path = _model_file_ctx.__enter__() _tagger.open(str(model_path)) return _tagger diff --git a/pythainlp/tokenize/nlpo3.py b/pythainlp/tokenize/nlpo3.py index 65734ea01..381f93ec2 100644 --- a/pythainlp/tokenize/nlpo3.py +++ b/pythainlp/tokenize/nlpo3.py @@ -14,7 +14,7 @@ _NLPO3_DEFAULT_DICT_NAME = "_73bcj049dzbu9t49b4va170k" # supposed to be unique _NLPO3_DEFAULT_DICT = None # Will be lazily loaded -_dict_path_ctx = None # Context manager kept alive for program lifetime +_dict_file_ctx = None # File context manager kept alive for program lifetime _load_lock = threading.Lock() # Thread safety for lazy loading @@ -25,15 +25,15 @@ def _ensure_default_dict_loaded(): The context manager is kept alive for the lifetime of the program to prevent cleanup of temporary files while the dictionary is in use. """ - global _NLPO3_DEFAULT_DICT, _dict_path_ctx + global _NLPO3_DEFAULT_DICT, _dict_file_ctx if _NLPO3_DEFAULT_DICT is None: with _load_lock: # Double-check pattern to avoid race conditions if _NLPO3_DEFAULT_DICT is None: corpus_files = files("pythainlp.corpus") dict_file = corpus_files.joinpath(_THAI_WORDS_FILENAME) - _dict_path_ctx = as_file(dict_file) - dict_path = _dict_path_ctx.__enter__() + _dict_file_ctx = as_file(dict_file) + dict_path = _dict_file_ctx.__enter__() _NLPO3_DEFAULT_DICT = nlpo3_load_dict( str(dict_path), _NLPO3_DEFAULT_DICT_NAME ) From c2526e3c5484dcc5e145d6bdd6f9c123d4996a53 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 17 Jan 2026 17:23:06 +0000 Subject: [PATCH 9/9] Use string-based import to avoid broad import Replace `import pythainlp; files(pythainlp)` with `files("pythainlp")` in get_pythainlp_path() to avoid importing the entire package unnecessarily. This is consistent with the pattern used elsewhere in the codebase and avoids potential circular import issues. Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/tools/path.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pythainlp/tools/path.py b/pythainlp/tools/path.py index 74e7cdcf8..821d268d1 100644 --- a/pythainlp/tools/path.py +++ b/pythainlp/tools/path.py @@ -77,8 +77,7 @@ def get_pythainlp_path() -> str: get_pythainlp_path() # output: '/usr/local/lib/python3.6/dist-packages/pythainlp' """ - import pythainlp - package_path = files(pythainlp) + package_path = files("pythainlp") # For compatibility, convert to string path if possible # This works for both regular installations and zip files if hasattr(package_path, '__fspath__'):