diff --git a/CHANGELOG.md b/CHANGELOG.md index 2af92d7d0..911ad15db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,16 @@ See PR for prompt and details. - Add BLEU, ROUGE, WER, and CER metrics to pythainlp.benchmarks #1295 - Add Attaparse engine to dependency parser (`dependency_parsing`, engine="attaparse") #1303 +- `get_corpus_path()` now respects `PYTHAINLP_OFFLINE` env var (same semantics + as `HF_HUB_OFFLINE`): when set, raises `FileNotFoundError` if the corpus is + not already cached locally; when unset, auto-downloads as before #1306 +- Added `pythainlp.is_offline_mode()` helper function (mirrors + `huggingface_hub.is_offline_mode()`) #1306 +- `PYTHAINLP_DATA` is now the preferred env var for the data directory + (same pattern as `NLTK_DATA`); `PYTHAINLP_DATA_DIR` is deprecated + and will emit a `DeprecationWarning` #1306 +- Callers raise `FileNotFoundError` with download instructions when a corpus + path cannot be resolved (e.g. download failed) #1306 - Improved documentation; code cleanup; more tests ## Version 5.1.2 -> 5.2.0 diff --git a/README.md b/README.md index 784fa67dc..3e5c5e0e7 100644 --- a/README.md +++ b/README.md @@ -77,14 +77,27 @@ please inspect the `[project.optional-dependencies]` section of PyThaiNLP downloads data (see the data catalog `db.json` at [pythainlp-corpus](https://github.com/PyThaiNLP/pythainlp-corpus)) to `~/pythainlp-data` by default. -Set the `PYTHAINLP_DATA_DIR` environment variable to override this location. +Set the `PYTHAINLP_DATA` environment variable to override this location. +(`PYTHAINLP_DATA_DIR` is still accepted but deprecated.) When using PyThaiNLP in distributed computing environments -(e.g., Apache Spark), set the `PYTHAINLP_DATA_DIR` environment variable +(e.g., Apache Spark), set the `PYTHAINLP_DATA` environment variable inside the function that will be distributed to worker nodes. See details in [the documentation](https://pythainlp.org/dev-docs/notes/installation.html). +### Offline mode + +Set `PYTHAINLP_OFFLINE=1` to disable automatic corpus downloads. +When this variable is set and a corpus is not already cached locally, +a `FileNotFoundError` is raised instead of attempting a network download. +Use `pythainlp.is_offline_mode()` to check the current state programmatically. + +```python +import pythainlp +print(pythainlp.is_offline_mode()) # True if PYTHAINLP_OFFLINE=1 +``` + ## Testing We test core functionalities on all officially supported Python versions. diff --git a/README_TH.md b/README_TH.md index 9c910674a..c0dd7f806 100644 --- a/README_TH.md +++ b/README_TH.md @@ -109,10 +109,11 @@ pip install "pythainlp[extra1,extra2,...]" PyThaiNLP ดาวน์โหลดข้อมูล (ดูแค็ตตาล็อกข้อมูล `db.json` ที่ [pythainlp-corpus](https://github.com/PyThaiNLP/pythainlp-corpus)) ไปที่ `~/pythainlp-data` ตามค่าเริ่มต้น -ตั้งค่า environment variable `PYTHAINLP_DATA_DIR` เพื่อเปลี่ยนตำแหน่งนี้ +ตั้งค่า environment variable `PYTHAINLP_DATA` เพื่อเปลี่ยนตำแหน่งนี้ +(`PYTHAINLP_DATA_DIR` ยังคงใช้ได้แต่เลิกใช้แล้ว) เมื่อใช้ PyThaiNLP ในสภาพแวดล้อมการคำนวณแบบกระจาย -(เช่น Apache Spark) ให้ตั้งค่า environment variable `PYTHAINLP_DATA_DIR` +(เช่น Apache Spark) ให้ตั้งค่า environment variable `PYTHAINLP_DATA` ภายในฟังก์ชันที่จะถูกกระจายไปยัง worker nodes ดูรายละเอียดใน[เอกสาร](https://pythainlp.org/dev-docs/notes/installation.html) diff --git a/docs/notes/installation.rst b/docs/notes/installation.rst index 1d6ad73a7..b2ba549bb 100644 --- a/docs/notes/installation.rst +++ b/docs/notes/installation.rst @@ -89,7 +89,8 @@ Key considerations 2. **Use a writable local directory**: The default data directory (``~/pythainlp-data``) may not be writable on executor nodes. Use a local directory like ``./pythainlp-data`` instead. -3. **Set ``PYTHAINLP_DATA_DIR`` before data access**: Always set the ``PYTHAINLP_DATA_DIR`` environment variable before the first call that reads or writes PyThaiNLP data on each worker. +3. **Set ``PYTHAINLP_DATA`` before data access**: Always set the ``PYTHAINLP_DATA`` environment variable before the first call that reads or writes PyThaiNLP data on each worker. + (``PYTHAINLP_DATA_DIR`` is also accepted for backward compatibility but is deprecated.) Example usage with Apache Spark ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -104,7 +105,7 @@ Basic example using PySpark RDD:: def tokenize_thai(text): import os - os.environ['PYTHAINLP_DATA_DIR'] = './pythainlp-data' + os.environ['PYTHAINLP_DATA'] = './pythainlp-data' from pythainlp.tokenize import word_tokenize return word_tokenize(text) @@ -123,7 +124,7 @@ Example using PySpark DataFrame API:: @udf(returnType=ArrayType(StringType())) def tokenize_udf(text): import os - os.environ['PYTHAINLP_DATA_DIR'] = './pythainlp-data' + os.environ['PYTHAINLP_DATA'] = './pythainlp-data' from pythainlp.tokenize import word_tokenize return word_tokenize(text) @@ -141,13 +142,30 @@ Note that while the code itself is thread-safe, you still need to configure the Runtime configurations ---------------------- -.. envvar:: PYTHAINLP_DATA_DIR +.. envvar:: PYTHAINLP_DATA Specifies the location where downloaded data and the corpus database are stored. If the directory does not exist, PyThaiNLP will create it. By default this is a directory named ``pythainlp-data`` in the user's home directory. - Run ``thainlp data path`` at the command line to display the current `PYTHAINLP_DATA_DIR`. + Run ``thainlp data path`` at the command line to display the current data directory. + +.. envvar:: PYTHAINLP_DATA_DIR + + .. deprecated:: + Use :envvar:`PYTHAINLP_DATA` instead. Setting ``PYTHAINLP_DATA_DIR`` triggers a + :class:`DeprecationWarning` at runtime. If both ``PYTHAINLP_DATA`` and ``PYTHAINLP_DATA_DIR`` + are set simultaneously, PyThaiNLP raises :exc:`ValueError`. + +.. envvar:: PYTHAINLP_OFFLINE + + When set to a truthy value (``1``, ``true``, ``yes``, ``on``), PyThaiNLP operates in + *offline mode*: corpus downloads are disabled, and :func:`pythainlp.corpus.get_corpus_path` + raises :exc:`FileNotFoundError` for any corpus that is not already cached locally. + + Use :func:`pythainlp.is_offline_mode` to check the current state programmatically. + + This follows the same convention as ``HF_HUB_OFFLINE`` in `huggingface_hub`. .. envvar:: PYTHAINLP_READ_MODE @@ -158,11 +176,11 @@ Installation FAQ Q: How do I set environment variables on each executor node in a distributed environment? -A: When using PyThaiNLP in distributed computing environments like Apache Spark, you need to set the ``PYTHAINLP_DATA_DIR`` environment variable inside the function that will be distributed to executor nodes. For example:: +A: When using PyThaiNLP in distributed computing environments like Apache Spark, you need to set the ``PYTHAINLP_DATA`` environment variable inside the function that will be distributed to executor nodes. For example:: def tokenize_thai(text): import os - os.environ['PYTHAINLP_DATA_DIR'] = './pythainlp-data' + os.environ['PYTHAINLP_DATA'] = './pythainlp-data' from pythainlp.tokenize import word_tokenize return word_tokenize(text) diff --git a/pythainlp/__init__.py b/pythainlp/__init__.py index c84e7bdda..01214514d 100644 --- a/pythainlp/__init__.py +++ b/pythainlp/__init__.py @@ -55,6 +55,7 @@ __all__: list[str] = [ "collate", "correct", + "is_offline_mode", "pos_tag", "romanize", "spell", @@ -76,5 +77,6 @@ subword_tokenize, word_tokenize, ) +from pythainlp.tools.path import is_offline_mode from pythainlp.transliterate import romanize, transliterate from pythainlp.util import collate, thai_strftime diff --git a/pythainlp/augment/word2vec/ltw2v.py b/pythainlp/augment/word2vec/ltw2v.py index 37918f91f..10745a078 100644 --- a/pythainlp/augment/word2vec/ltw2v.py +++ b/pythainlp/augment/word2vec/ltw2v.py @@ -32,10 +32,12 @@ def tokenizer(self, text: str) -> list[str]: def load_w2v(self) -> None: # insert substitute """Load LTW2V's word2vec model""" - if self.ltw2v_wv is None: - raise ValueError( - "LTW2V word2vec model not found. " - "Please download it first using pythainlp.corpus.download('ltw2v_wv')" + if not self.ltw2v_wv: + raise FileNotFoundError( + "corpus-not-found name='ltw2v_wv'\n" + " Corpus 'ltw2v_wv' not found.\n" + " Python: pythainlp.corpus.download('ltw2v_wv')\n" + " CLI: thainlp data get ltw2v_wv" ) self.aug: Word2VecAug = Word2VecAug( self.ltw2v_wv, self.tokenizer, type="binary" diff --git a/pythainlp/augment/word2vec/thai2fit.py b/pythainlp/augment/word2vec/thai2fit.py index a6aaa301e..5f34937af 100644 --- a/pythainlp/augment/word2vec/thai2fit.py +++ b/pythainlp/augment/word2vec/thai2fit.py @@ -33,10 +33,12 @@ def tokenizer(self, text: str) -> list[str]: def load_w2v(self) -> None: """Load Thai2Fit's word2vec model""" - if self.thai2fit_wv is None: - raise ValueError( - "Thai2Fit word2vec model not found. " - "Please download it first using pythainlp.corpus.download('thai2fit_wv')" + if not self.thai2fit_wv: + raise FileNotFoundError( + "corpus-not-found name='thai2fit_wv'\n" + " Corpus 'thai2fit_wv' not found.\n" + " Python: pythainlp.corpus.download('thai2fit_wv')\n" + " CLI: thainlp data get thai2fit_wv" ) self.aug: Word2VecAug = Word2VecAug( self.thai2fit_wv, self.tokenizer, type="binary" diff --git a/pythainlp/cli/data.py b/pythainlp/cli/data.py index 7ff9fcef3..175732b62 100644 --- a/pythainlp/cli/data.py +++ b/pythainlp/cli/data.py @@ -33,7 +33,8 @@ def __init__(self, argv: Sequence[str]) -> None: "Current data path:\n\n" f"{get_pythainlp_data_path()}\n\n" "To change PyThaiNLP data path, set the operating system's\n" - "PYTHAINLP_DATA_DIR environment variable.\n\n" + "PYTHAINLP_DATA environment variable.\n" + "(PYTHAINLP_DATA_DIR is also accepted but deprecated.)\n\n" "For more information about corpora that PyThaiNLP use, see:\n" "https://github.com/PyThaiNLP/pythainlp-corpus/\n\n" "--" diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index d774652c3..403c00c84 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -18,6 +18,7 @@ from pythainlp import __version__ from pythainlp.corpus import corpus_db_path, corpus_db_url, corpus_path from pythainlp.tools import get_full_data_path +from pythainlp.tools.path import is_offline_mode if TYPE_CHECKING: from typing import Any, Optional @@ -245,46 +246,74 @@ def get_corpus_default_db(name: str, version: str = "") -> Optional[str]: return None -def get_corpus_path( - name: str, version: str = "", force: bool = False +def _resolve_corpus_file_path( + corpus_db_detail: dict[str, Any], ) -> Optional[str]: + """Resolve the local filesystem path for a corpus catalog entry. + + :param dict corpus_db_detail: a corpus catalog entry from the local DB + :return: full local path to the corpus file or folder, + or ``None`` if required path information is missing + :rtype: Optional[str] + """ + if corpus_db_detail.get("is_folder"): + foldername = corpus_db_detail.get("foldername") + return get_full_data_path(foldername) if foldername else None + filename = corpus_db_detail.get("filename") + return get_full_data_path(filename) if filename else None + + +def get_corpus_path(name: str, version: str = "") -> Optional[str]: """Get corpus path. + The function checks the following locations in order: + + 1. A user-defined path override (``CUSTOMIZE`` mapping). + 2. Bundled (default) corpora shipped with PyThaiNLP. + 3. The local download catalog (``~/pythainlp-data/``). + + When the corpus file is not present locally, the behavior depends on the + ``PYTHAINLP_OFFLINE`` environment variable: + + - If ``PYTHAINLP_OFFLINE`` is set to a truthy value (e.g., ``"1"``), + a :exc:`FileNotFoundError` is raised immediately. + - Otherwise, the corpus is downloaded automatically. + :param str name: corpus name - :param str version: version - :param bool force: force downloading - :return: path to the corpus or **None** if the corpus doesn't \ - exist on the device - :rtype: str + :param str version: corpus version (empty string means latest) + :return: full local path when the corpus exists, + or ``None`` when the corpus cannot be found or downloaded. + :rtype: Optional[str] + + :raises FileNotFoundError: when the corpus is missing locally and + ``PYTHAINLP_OFFLINE`` is set to a truthy value. :Example: (Please see the filename in - `this file - `_ + `this file `_) If the corpus already exists:: from pythainlp.corpus import get_corpus_path - print(get_corpus_path('ttc')) + print(get_corpus_path("ttc")) # output: /root/pythainlp-data/ttc_freq.txt - If the corpus has not been downloaded yet:: + If the corpus has not been downloaded yet (online mode):: - from pythainlp.corpus import download, get_corpus_path + from pythainlp.corpus import get_corpus_path - print(get_corpus_path('wiki_lm_lstm')) - # output: None + print(get_corpus_path("wiki_lm_lstm")) + # output: /root/pythainlp-data/thwiki_model_lstm.pth + # (downloads automatically on first call) - download('wiki_lm_lstm') - # output: - # Download: wiki_lm_lstm - # wiki_lm_lstm 0.32 - # thwiki_lm.pth?dl=1: 1.05GB [00:25, 41.5MB/s] - # /root/pythainlp-data/thwiki_model_lstm.pth + To download manually:: - print(get_corpus_path('wiki_lm_lstm')) + from pythainlp.corpus import download, get_corpus_path + + download("wiki_lm_lstm") + print(get_corpus_path("wiki_lm_lstm")) # output: /root/pythainlp-data/thwiki_model_lstm.pth """ CUSTOMIZE: dict[str, str] = { @@ -293,38 +322,50 @@ def get_corpus_path( if name in CUSTOMIZE: return CUSTOMIZE[name] + # Check bundled (default) corpora first default_path = get_corpus_default_db(name=name, version=version) if default_path is not None: return default_path - # check if the corpus is in local catalog, download it if not + # Check the local download catalog corpus_db_detail = get_corpus_db_detail(name, version=version) - - if not corpus_db_detail or not corpus_db_detail.get("filename"): - download(name, version=version, force=force) + if not corpus_db_detail: + # Corpus not in local catalog; download it unless in offline mode + if is_offline_mode(): + raise FileNotFoundError( + f"corpus-not-found name={name!r}\n" + f" Corpus '{name}' not found locally.\n" + f" PYTHAINLP_OFFLINE is set; automatic downloading is disabled.\n" + f" To download, unset PYTHAINLP_OFFLINE, then run:\n" + f" Python: pythainlp.corpus.download('{name}')\n" + f" CLI: thainlp data get {name}" + ) + if not download(name, version=version): + return None corpus_db_detail = get_corpus_db_detail(name, version=version) - - if corpus_db_detail and corpus_db_detail.get("filename"): - # corpus is in the local catalog, get full path to the file - if corpus_db_detail.get("is_folder"): - foldername = corpus_db_detail.get("foldername") - if foldername: - path = get_full_data_path(foldername) - else: - return None - else: - filename = corpus_db_detail.get("filename") - if filename: - path = get_full_data_path(filename) - else: - return None - # check if the corpus file actually exists, download it if not - if not os.path.exists(path): - download(name, version=version, force=force) - if os.path.exists(path): - return path - - return None + if not corpus_db_detail: + return None + + path = _resolve_corpus_file_path(corpus_db_detail) + if path is None: + return None + + if os.path.exists(path): + return path + + # File is registered in catalog but missing from disk + if is_offline_mode(): + raise FileNotFoundError( + f"corpus-not-found name={name!r} expected-path={path!r}\n" + f" Corpus '{name}' expected at '{path}' but file not found.\n" + f" PYTHAINLP_OFFLINE is set; automatic re-downloading is disabled.\n" + f" To re-download, unset PYTHAINLP_OFFLINE, then run:\n" + f" Python: pythainlp.corpus.download('{name}', force=True)\n" + f" CLI: thainlp data get {name}" + ) + if not download(name, version=version, force=True): + return None + return path if os.path.exists(path) else None def _download(url: str, dst: str) -> int: @@ -588,6 +629,12 @@ def download( The available corpus names can be seen in this file: https://pythainlp.org/pythainlp-corpus/db.json + This function always performs the download regardless of the + ``PYTHAINLP_OFFLINE`` environment variable, because an explicit call + to ``download()`` is a deliberate user action. + ``PYTHAINLP_OFFLINE`` only blocks the *automatic* download triggered + by :func:`pythainlp.corpus.get_corpus_path`. + :param str name: corpus name :param bool force: force downloading :param str url: URL of the corpus catalog @@ -614,6 +661,7 @@ def download( if _CHECK_MODE == "1": print("PyThaiNLP is read-only mode. It can't download.") return False + if not url: url = corpus_db_url() @@ -788,8 +836,13 @@ def remove(name: str) -> bool: def get_path_folder_corpus(name: str, version: str, *path: str) -> str: corpus_path = get_corpus_path(name, version) - if corpus_path is None: - raise ValueError(f"Corpus path not found for {name} version {version}") + if not corpus_path: + raise FileNotFoundError( + f"corpus-not-found name={name!r} version={version!r}\n" + f" Corpus '{name}' (version {version}) not found.\n" + f" Python: pythainlp.corpus.download('{name}')\n" + f" CLI: thainlp data get {name}" + ) return os.path.join(corpus_path, *path) diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index aab4d6ac5..1c260eeff 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -59,11 +59,13 @@ # Validate that corpus files are available if thwiki["itos_fname"] is None or thwiki["wgts_fname"] is None: - raise RuntimeError( - "Thai2fit model files not found. " - "Please download the corpus first:\n" - " pythainlp.corpus.download('wiki_lm_lstm')\n" - " pythainlp.corpus.download('wiki_itos_lstm')" + raise FileNotFoundError( + "corpus-not-found names=['wiki_lm_lstm', 'wiki_itos_lstm']\n" + " Thai2fit model files not found.\n" + " Python: pythainlp.corpus.download('wiki_lm_lstm')\n" + " CLI: thainlp data get wiki_lm_lstm\n" + " Python: pythainlp.corpus.download('wiki_itos_lstm')\n" + " CLI: thainlp data get wiki_itos_lstm" ) # Security Note: This loads a pickle file from PyThaiNLP's trusted corpus. diff --git a/pythainlp/spell/symspellpy.py b/pythainlp/spell/symspellpy.py index a11e1deb8..54ee3246f 100644 --- a/pythainlp/spell/symspellpy.py +++ b/pythainlp/spell/symspellpy.py @@ -65,8 +65,16 @@ def _get_sym_spell() -> SymSpell: encoding="utf-8-sig", ) # Load bigram dictionary from downloaded corpus + bigram_path = get_corpus_path(_BIGRAM_CORPUS_NAME) + if not bigram_path: + raise FileNotFoundError( + f"corpus-not-found name={_BIGRAM_CORPUS_NAME!r}\n" + f" Corpus '{_BIGRAM_CORPUS_NAME}' not found.\n" + f" Python: pythainlp.corpus.download('{_BIGRAM_CORPUS_NAME}')\n" + f" CLI: thainlp data get {_BIGRAM_CORPUS_NAME}" + ) _sym_spell.load_bigram_dictionary( - get_corpus_path(_BIGRAM_CORPUS_NAME), + bigram_path, 0, 2, separator="\t", diff --git a/pythainlp/tag/perceptron.py b/pythainlp/tag/perceptron.py index 98ebc0817..ab7c980ad 100644 --- a/pythainlp/tag/perceptron.py +++ b/pythainlp/tag/perceptron.py @@ -50,8 +50,13 @@ def _blackboard_tagger() -> PerceptronTagger: global _BLACKBOARD_TAGGER if not _BLACKBOARD_TAGGER: path = get_corpus_path(_BLACKBOARD_NAME) - if path is None: - raise ValueError(f"Corpus path not found for {_BLACKBOARD_NAME}") + if not path: + raise FileNotFoundError( + f"corpus-not-found name={_BLACKBOARD_NAME!r}\n" + f" Corpus '{_BLACKBOARD_NAME}' not found.\n" + f" Python: pythainlp.corpus.download('{_BLACKBOARD_NAME}')\n" + f" CLI: thainlp data get {_BLACKBOARD_NAME}" + ) _BLACKBOARD_TAGGER = PerceptronTagger(path=path) return _BLACKBOARD_TAGGER diff --git a/pythainlp/tag/thai_nner.py b/pythainlp/tag/thai_nner.py index 2ff27b40a..60d945db5 100644 --- a/pythainlp/tag/thai_nner.py +++ b/pythainlp/tag/thai_nner.py @@ -120,6 +120,13 @@ def __init__(self, path_model: Optional[str] = None) -> None: # Resolve path_model at runtime to avoid freezing the value at module import time if path_model is None: path_model = get_corpus_path("thai_nner", "1.0") + if not path_model: + raise FileNotFoundError( + "corpus-not-found name='thai_nner'\n" + " Corpus 'thai_nner' not found.\n" + " Python: pythainlp.corpus.download('thai_nner')\n" + " CLI: thainlp data get thai_nner" + ) # Import inside __init__ (not at module level) to allow: # 1. Helper functions (get_top_level_entities, _entities_to_iob, etc.) to work diff --git a/pythainlp/tag/thainer.py b/pythainlp/tag/thainer.py index 012b88a36..63f804ca2 100644 --- a/pythainlp/tag/thainer.py +++ b/pythainlp/tag/thainer.py @@ -108,21 +108,23 @@ def __init__(self, version: str = "1.4") -> None: if version == "1.4": model_path = get_corpus_path("thainer-1.4", version="1.4") - if model_path is None: - raise RuntimeError( - "ThaiNER 1.4 model not found. " - "Please download the corpus first:\n" - " pythainlp.corpus.download('thainer-1.4')" + if not model_path: + raise FileNotFoundError( + "corpus-not-found name='thainer-1.4'\n" + " Corpus 'thainer-1.4' not found.\n" + " Python: pythainlp.corpus.download('thainer-1.4')\n" + " CLI: thainlp data get thainer-1.4" ) self.crf.open(model_path) self.pos_tag_name: str = "orchid_ud" elif version == "1.5": model_path = get_corpus_path("thainer", version="1.5") - if model_path is None: - raise RuntimeError( - "ThaiNER 1.5 model not found. " - "Please download the corpus first:\n" - " pythainlp.corpus.download('thainer')" + if not model_path: + raise FileNotFoundError( + "corpus-not-found name='thainer'\n" + " Corpus 'thainer' not found.\n" + " Python: pythainlp.corpus.download('thainer')\n" + " CLI: thainlp data get thainer" ) self.crf.open(model_path) self.pos_tag_name = "blackboard" diff --git a/pythainlp/tag/unigram.py b/pythainlp/tag/unigram.py index b6ed243a2..88504167f 100644 --- a/pythainlp/tag/unigram.py +++ b/pythainlp/tag/unigram.py @@ -53,8 +53,13 @@ def _blackboard_tagger() -> dict[str, str]: global _BLACKBOARD_TAGGER if not _BLACKBOARD_TAGGER: path = get_corpus_path(_BLACKBOARD_NAME) - if path is None: - raise ValueError(f"Corpus path not found for {_BLACKBOARD_NAME}") + if not path: + raise FileNotFoundError( + f"corpus-not-found name={_BLACKBOARD_NAME!r}\n" + f" Corpus '{_BLACKBOARD_NAME}' not found.\n" + f" Python: pythainlp.corpus.download('{_BLACKBOARD_NAME}')\n" + f" CLI: thainlp data get {_BLACKBOARD_NAME}" + ) with open(path, encoding="utf-8-sig") as fh: _BLACKBOARD_TAGGER = json.load(fh) return _BLACKBOARD_TAGGER diff --git a/pythainlp/tools/__init__.py b/pythainlp/tools/__init__.py index acc6502ff..9e68d0b4d 100644 --- a/pythainlp/tools/__init__.py +++ b/pythainlp/tools/__init__.py @@ -6,6 +6,7 @@ "get_full_data_path", "get_pythainlp_data_path", "get_pythainlp_path", + "is_offline_mode", "safe_print", "warn_deprecation", ] @@ -16,4 +17,5 @@ get_full_data_path, get_pythainlp_data_path, get_pythainlp_path, + is_offline_mode, ) diff --git a/pythainlp/tools/path.py b/pythainlp/tools/path.py index 0ab6ac35b..7459bce2d 100644 --- a/pythainlp/tools/path.py +++ b/pythainlp/tools/path.py @@ -21,6 +21,47 @@ PYTHAINLP_DEFAULT_DATA_DIR: str = "pythainlp-data" +def is_offline_mode() -> bool: + """Return whether PyThaiNLP is operating in offline mode. + + Offline mode is activated by setting the ``PYTHAINLP_OFFLINE`` + environment variable to a truthy value (e.g. ``"1"``). + Falsy values (``""``, ``"0"``, ``"false"``, ``"no"``, ``"off"``) + keep online mode active. + + This follows the same convention as ``HF_HUB_OFFLINE`` in + `huggingface_hub`. + + When offline mode is active, :func:`pythainlp.corpus.get_corpus_path` + raises :exc:`FileNotFoundError` for any corpus that is not already + cached locally, instead of triggering an automatic download. + + .. note:: + :func:`pythainlp.corpus.download` always executes regardless of + this setting, because an explicit call to ``download()`` or + ``thainlp data get`` is a deliberate user action. + ``PYTHAINLP_OFFLINE`` only prevents *automatic* downloads + initiated by :func:`~pythainlp.corpus.get_corpus_path`. + + :return: ``True`` if PyThaiNLP is in offline mode, ``False`` otherwise. + :rtype: bool + + :Example: + :: + + import os + from pythainlp import is_offline_mode + + os.environ["PYTHAINLP_OFFLINE"] = "1" + print(is_offline_mode()) # True + + os.environ["PYTHAINLP_OFFLINE"] = "0" + print(is_offline_mode()) # False + """ + val = os.getenv("PYTHAINLP_OFFLINE", "") + return val.strip().lower() not in ("", "0", "false", "no", "off") + + def get_full_data_path(path: str) -> str: """This function joins path of :mod:`pythainlp` data directory and the given path, and returns the full path. @@ -40,11 +81,24 @@ def get_full_data_path(path: str) -> str: def get_pythainlp_data_path() -> str: - """Returns the full path where PyThaiNLP keeps its (downloaded) data. - If the directory does not yet exist, it will be created. - The path can be specified through the environment variable - :envvar:`PYTHAINLP_DATA_DIR`. By default, `~/pythainlp-data` - will be used. + """Return the full path where PyThaiNLP keeps its (downloaded) data. + + The directory is created if it does not yet exist. + + The path is resolved in the following order: + + 1. ``PYTHAINLP_DATA`` environment variable (preferred). + 2. ``PYTHAINLP_DATA_DIR`` environment variable + (deprecated; shows a warning). + 3. If **both** variables are set, the function raises + :exc:`ValueError` because the conflict must be resolved + explicitly. + 4. If neither is set, ``~/pythainlp-data`` is used. + + .. deprecated:: + ``PYTHAINLP_DATA_DIR`` is deprecated. + Use ``PYTHAINLP_DATA`` instead (follows the same pattern as + ``NLTK_DATA``). :return: full path of directory for :mod:`pythainlp` downloaded data :rtype: str @@ -57,10 +111,27 @@ def get_pythainlp_data_path() -> str: get_pythainlp_data_path() # output: '/root/pythainlp-data' """ - pythainlp_data_dir = os.getenv( - "PYTHAINLP_DATA_DIR", os.path.join("~", PYTHAINLP_DEFAULT_DATA_DIR) - ) - path = os.path.expanduser(pythainlp_data_dir) + import warnings + + data_dir = os.getenv("PYTHAINLP_DATA") + data_dir_legacy = os.getenv("PYTHAINLP_DATA_DIR") + + if data_dir and data_dir_legacy: + raise ValueError( + "Both PYTHAINLP_DATA and PYTHAINLP_DATA_DIR are set. " + "Please use PYTHAINLP_DATA only and unset PYTHAINLP_DATA_DIR." + ) + + if data_dir_legacy and not data_dir: + warnings.warn( + "PYTHAINLP_DATA_DIR is deprecated; use PYTHAINLP_DATA instead.", + DeprecationWarning, + stacklevel=2, + ) + data_dir = data_dir_legacy + + resolved = data_dir or os.path.join("~", PYTHAINLP_DEFAULT_DATA_DIR) + path = os.path.expanduser(resolved) os.makedirs(path, exist_ok=True) return path diff --git a/pythainlp/translate/en_th.py b/pythainlp/translate/en_th.py index ecf370156..36f92efd7 100644 --- a/pythainlp/translate/en_th.py +++ b/pythainlp/translate/en_th.py @@ -43,13 +43,13 @@ def _get_translate_path(model: str, *path: str) -> str: corpus_path = get_corpus_path(model, version="1.0") - if corpus_path is None: + if not corpus_path: return "" return os.path.join(corpus_path, *path) def _download_install(name: str) -> None: - if get_corpus_path(name) is None: + if not get_corpus_path(name): download(name, force=True, version="1.0") diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index 2715cc3c0..bbec2d261 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -40,8 +40,14 @@ def __init__(self) -> None: Now supports Thai to Latin (romanization) """ - # get the model, download it if it's not available locally self.__model_filename = get_corpus_path(_MODEL_NAME) # type: ignore[assignment] + if not self.__model_filename: + raise FileNotFoundError( + f"corpus-not-found name={_MODEL_NAME!r}\n" + f" Corpus '{_MODEL_NAME}' not found.\n" + f" Python: pythainlp.corpus.download('{_MODEL_NAME}')\n" + f" CLI: thainlp data get {_MODEL_NAME}" + ) loader = torch.load(self.__model_filename, map_location=device) @@ -57,9 +63,7 @@ def __init__(self) -> None: # encoder/ decoder # Restore the model and construct the encoder and decoder. - self._encoder = Encoder( - INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT - ) + self._encoder = Encoder(INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT) self._decoder = AttentionDecoder( OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT @@ -209,9 +213,7 @@ def __init__(self, method: str, hidden_size: int) -> None: elif self.method == "concat": self.attn = nn.Linear(self.hidden_size * 2, hidden_size) - self.other = nn.Parameter( - torch.FloatTensor(1, hidden_size) - ) + self.other = nn.Parameter(torch.FloatTensor(1, hidden_size)) def forward( self, diff --git a/pythainlp/transliterate/thai2rom_onnx.py b/pythainlp/transliterate/thai2rom_onnx.py index 64c41b065..c2056ed4b 100644 --- a/pythainlp/transliterate/thai2rom_onnx.py +++ b/pythainlp/transliterate/thai2rom_onnx.py @@ -32,6 +32,30 @@ def __init__(self) -> None: self.__encoder_filename: str = get_corpus_path(_MODEL_ENCODER_NAME) # type: ignore[assignment] self.__decoder_filename: str = get_corpus_path(_MODEL_DECODER_NAME) # type: ignore[assignment] self.__config_filename: str = get_corpus_path(_MODEL_CONFIG_NAME) # type: ignore[assignment] + if ( + not self.__encoder_filename + or not self.__decoder_filename + or not self.__config_filename + ): + missing = [ + n + for n, v in ( + (_MODEL_ENCODER_NAME, self.__encoder_filename), + (_MODEL_DECODER_NAME, self.__decoder_filename), + (_MODEL_CONFIG_NAME, self.__config_filename), + ) + if not v + ] + raise FileNotFoundError( + f"corpus-not-found names={missing!r}\n" + f" Corpus file(s) not found: {', '.join(missing)}.\n" + f" Download each missing corpus:\n" + + "\n".join( + f" Python: pythainlp.corpus.download('{n}')\n" + f" CLI: thainlp data get {n}" + for n in missing + ) + ) # loader = torch.load(self.__model_filename, map_location=device) with open(str(self.__config_filename)) as f: diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index 125a96157..b4fa830ab 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -49,8 +49,14 @@ class ThaiG2P: _network: "Seq2Seq" def __init__(self) -> None: - # get the model, download it if it's not available locally self.__model_filename: str = get_corpus_path(_MODEL_NAME) # type: ignore[assignment] + if not self.__model_filename: + raise FileNotFoundError( + f"corpus-not-found name={_MODEL_NAME!r}\n" + f" Corpus '{_MODEL_NAME}' not found.\n" + f" Python: pythainlp.corpus.download('{_MODEL_NAME}')\n" + f" CLI: thainlp data get {_MODEL_NAME}" + ) loader = torch.load(self.__model_filename, map_location=device) diff --git a/pythainlp/transliterate/w2p.py b/pythainlp/transliterate/w2p.py index 9ad03c49a..c863f5b35 100644 --- a/pythainlp/transliterate/w2p.py +++ b/pythainlp/transliterate/w2p.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Optional -from pythainlp.corpus import download, get_corpus_path +from pythainlp.corpus import get_corpus_path if TYPE_CHECKING: import numpy as np @@ -87,13 +87,13 @@ def __init__(self) -> None: self.checkpoint: Optional[str] = get_corpus_path( _MODEL_NAME, version="0.2" ) - if self.checkpoint is None: - download(_MODEL_NAME, version="0.2") - self.checkpoint = get_corpus_path(_MODEL_NAME) - if self.checkpoint is None: - raise RuntimeError( - f"Failed to download or locate {_MODEL_NAME} corpus" - ) + if not self.checkpoint: + raise FileNotFoundError( + f"corpus-not-found name={_MODEL_NAME!r}\n" + f" Corpus '{_MODEL_NAME}' not found.\n" + f" Python: pythainlp.corpus.download('{_MODEL_NAME}', version='0.2')\n" + f" CLI: thainlp data get {_MODEL_NAME}" + ) self._load_variables() def _load_variables(self) -> None: diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index c6a85f62f..23af19986 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -66,12 +66,14 @@ def get_thwiki_lstm() -> dict[str, str]: wgts_fname = THWIKI_LSTM["wgts_fname"] itos_fname = THWIKI_LSTM["itos_fname"] - if wgts_fname is None or itos_fname is None: - raise RuntimeError( - "ULMFiT model files not found. " - "Please download the corpus first:\n" - " pythainlp.corpus.download('wiki_lm_lstm')\n" - " pythainlp.corpus.download('wiki_itos_lstm')" + if not wgts_fname or not itos_fname: + raise FileNotFoundError( + "corpus-not-found names=['wiki_lm_lstm', 'wiki_itos_lstm']\n" + " ULMFiT model files not found.\n" + " Python: pythainlp.corpus.download('wiki_lm_lstm')\n" + " CLI: thainlp data get wiki_lm_lstm\n" + " Python: pythainlp.corpus.download('wiki_itos_lstm')\n" + " CLI: thainlp data get wiki_itos_lstm" ) return {"wgts_fname": wgts_fname, "itos_fname": itos_fname} diff --git a/pythainlp/word_vector/core.py b/pythainlp/word_vector/core.py index 119517b5d..67e748d21 100644 --- a/pythainlp/word_vector/core.py +++ b/pythainlp/word_vector/core.py @@ -58,8 +58,16 @@ def load_wordvector(self, model_name: str) -> None: from gensim.models import KeyedVectors self.model_name = model_name + corpus_file = get_corpus_path(self.model_name) + if not corpus_file: + raise FileNotFoundError( + f"corpus-not-found name={model_name!r}\n" + f" Corpus '{model_name}' not found.\n" + f" Python: pythainlp.corpus.download('{model_name}')\n" + f" CLI: thainlp data get {model_name}" + ) self.model = KeyedVectors.load_word2vec_format( - get_corpus_path(self.model_name), + corpus_file, binary=True, unicode_errors="ignore", ) diff --git a/tests/core/test_corpus.py b/tests/core/test_corpus.py index a90ae2789..c8447f259 100644 --- a/tests/core/test_corpus.py +++ b/tests/core/test_corpus.py @@ -125,8 +125,10 @@ def test_oscar(self): """ mock_path = "/mock/path/oscar_icu" - with patch('pythainlp.corpus.oscar.get_corpus_path', return_value=mock_path): - with patch('builtins.open', mock_open(read_data=mock_oscar_data)): + with patch( + "pythainlp.corpus.oscar.get_corpus_path", return_value=mock_path + ): + with patch("builtins.open", mock_open(read_data=mock_oscar_data)): result = oscar.word_freqs() self.assertIsNotNone(result) self.assertIsInstance(result, list) @@ -140,7 +142,9 @@ def test_oscar(self): self.assertNotIn('"', word) # Reset mock for unigram test - with patch('builtins.open', mock_open(read_data=mock_oscar_data)): + with patch( + "builtins.open", mock_open(read_data=mock_oscar_data) + ): result_unigram = oscar.unigram_word_freqs() self.assertIsNotNone(result_unigram) self.assertIsInstance(result_unigram, dict) @@ -164,7 +168,10 @@ def test_tnc(self): ภาษา ไทย คน 3""" # Test unigram functions - with patch('pythainlp.corpus.tnc.get_corpus', return_value=frozenset(mock_unigram_data.split('\n'))): + with patch( + "pythainlp.corpus.tnc.get_corpus", + return_value=frozenset(mock_unigram_data.split("\n")), + ): result = tnc.word_freqs() self.assertIsNotNone(result) self.assertIsInstance(result, list) @@ -180,8 +187,11 @@ def test_tnc(self): # Test bigram function mock_bigram_path = "/mock/path/bigram" - with patch('pythainlp.corpus.tnc.get_corpus_path', return_value=mock_bigram_path): - with patch('builtins.open', mock_open(read_data=mock_bigram_data)): + with patch( + "pythainlp.corpus.tnc.get_corpus_path", + return_value=mock_bigram_path, + ): + with patch("builtins.open", mock_open(read_data=mock_bigram_data)): result_bigram = tnc.bigram_word_freqs() self.assertIsNotNone(result_bigram) self.assertIsInstance(result_bigram, dict) @@ -190,8 +200,13 @@ def test_tnc(self): # Test trigram function mock_trigram_path = "/mock/path/trigram" - with patch('pythainlp.corpus.tnc.get_corpus_path', return_value=mock_trigram_path): - with patch('builtins.open', mock_open(read_data=mock_trigram_data)): + with patch( + "pythainlp.corpus.tnc.get_corpus_path", + return_value=mock_trigram_path, + ): + with patch( + "builtins.open", mock_open(read_data=mock_trigram_data) + ): result_trigram = tnc.trigram_word_freqs() self.assertIsNotNone(result_trigram) self.assertIsInstance(result_trigram, dict) @@ -217,8 +232,8 @@ def test_phupha(self): self.assertGreater(len(unigram_result), 0) # Check that common Thai words exist - self.assertIn('ไทย', unigram_result) - self.assertGreater(unigram_result['ไทย'], 0) + self.assertIn("ไทย", unigram_result) + self.assertGreater(unigram_result["ไทย"], 0) # Verify the full dataset is available (not pre-filtered) # The full dataset should have more words than just ORST @@ -230,7 +245,10 @@ def test_ttc(self): ไทย 500 ภาษา 300""" - with patch('pythainlp.corpus.ttc.get_corpus', return_value=frozenset(mock_ttc_data.split('\n'))): + with patch( + "pythainlp.corpus.ttc.get_corpus", + return_value=frozenset(mock_ttc_data.split("\n")), + ): result = ttc.word_freqs() self.assertIsNotNone(result) self.assertIsInstance(result, list) @@ -244,6 +262,59 @@ def test_ttc(self): self.assertGreater(len(result_unigram), 0) self.assertEqual(result_unigram["คน"], 1000) + def test_get_corpus_path_offline_mode(self): + """Test get_corpus_path() behavior with PYTHAINLP_OFFLINE env var.""" + # Unknown corpus name: download is attempted (it fails) → None + with patch.dict(os.environ, {"PYTHAINLP_OFFLINE": ""}): + self.assertIsNone(get_corpus_path("XXXkdjfBzc_nonexistent")) + + # When PYTHAINLP_OFFLINE=1 and corpus not in local catalog → raises + with patch.dict(os.environ, {"PYTHAINLP_OFFLINE": "1"}): + with patch( + "pythainlp.corpus.core.get_corpus_db_detail", + return_value={}, + ): + with self.assertRaises(FileNotFoundError) as ctx: + get_corpus_path("some_corpus") + self.assertIn("PYTHAINLP_OFFLINE", str(ctx.exception)) + + # When PYTHAINLP_OFFLINE=1 and file registered but missing → raises + fake_db_detail = {"name": "fake_corpus", "filename": "fake_file.txt"} + with patch.dict(os.environ, {"PYTHAINLP_OFFLINE": "1"}): + with patch( + "pythainlp.corpus.core.get_corpus_db_detail", + return_value=fake_db_detail, + ): + with patch("os.path.exists", return_value=False): + with self.assertRaises(FileNotFoundError) as ctx: + get_corpus_path("fake_corpus") + self.assertIn("PYTHAINLP_OFFLINE", str(ctx.exception)) + + # When PYTHAINLP_OFFLINE=1 and file exists → returns path normally + with patch.dict(os.environ, {"PYTHAINLP_OFFLINE": "1"}): + with patch( + "pythainlp.corpus.core.get_corpus_db_detail", + return_value=fake_db_detail, + ): + with patch("os.path.exists", return_value=True): + result = get_corpus_path("fake_corpus") + self.assertIsNotNone(result) + self.assertNotEqual(result, "") + + def test_download_ignores_offline_mode(self): + """download() must work even when PYTHAINLP_OFFLINE=1. + + Explicit calls to download() are deliberate user actions and must + not be blocked by the PYTHAINLP_OFFLINE environment variable. + That variable only prevents the *automatic* download triggered by + get_corpus_path() when a corpus is missing locally. + """ + # Use the real "test" corpus so the download actually goes through + with patch.dict(os.environ, {"PYTHAINLP_OFFLINE": "1"}): + result = download("test") + # Should succeed (returns True), not be blocked + self.assertTrue(result) + def test_revise_wordset(self): training_data = [ ["ถวิล อุดล", " ", "เป็น", "นักการเมือง", "หนึ่ง", "ใน"], @@ -255,7 +326,8 @@ def test_revise_wordset(self): self.assertIsInstance(revise_newmm_default_wordset(training_data), set) def test_zip(self): - p = get_corpus_path("test_zip") # may need to reduce the test zip size + self.assertTrue(download("test_zip")) # download first + p = get_corpus_path("test_zip") self.assertTrue(os.path.isdir(p)) self.assertTrue(remove("test_zip")) diff --git a/tests/core/test_generate.py b/tests/core/test_generate.py index 03d6f399d..5d601d5fc 100644 --- a/tests/core/test_generate.py +++ b/tests/core/test_generate.py @@ -4,10 +4,18 @@ import unittest +from pythainlp.corpus import download from pythainlp.generate import Bigram, Trigram, Unigram class GenerateTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + """Download required corpora before running tests.""" + download("oscar_icu") + download("tnc_bigram_word_freqs") + download("tnc_trigram_word_freqs") + def test_unigram(self): tnc_unigram = Unigram("tnc") self.assertIsNotNone(tnc_unigram.gen_sentence()) diff --git a/tests/core/test_tag.py b/tests/core/test_tag.py index 7378b45b3..95e25e03f 100644 --- a/tests/core/test_tag.py +++ b/tests/core/test_tag.py @@ -5,6 +5,7 @@ import unittest from os import path +from pythainlp.corpus import download from pythainlp.tag import ( NER, PerceptronTagger, @@ -21,6 +22,11 @@ class TagTestCase(unittest.TestCase): """Test pythainlp.tag.pos_tag""" + @classmethod + def setUpClass(cls) -> None: + """Download required corpora before running tests.""" + download("blackboard_unigram_tagger") + def test_pos_tag(self): self.assertEqual(pos_tag(None), []) self.assertEqual(pos_tag([]), []) @@ -105,6 +111,11 @@ class PerceptronTaggerTestCase(unittest.TestCase): :type unittest: _type_ """ + @classmethod + def setUpClass(cls) -> None: + """Download required corpora before running tests.""" + download("blackboard_pt_tagger") + def test_perceptron_tagger(self): self.assertEqual(perceptron.tag(None, corpus="orchid"), []) self.assertEqual(perceptron.tag([], corpus="orchid"), []) @@ -225,20 +236,20 @@ def test_get_top_level_entities(self): # Test with nested entities entities = [ - {'text': ['ห้า'], 'span': [7, 9], 'entity_type': 'cardinal'}, - {'text': ['ห้า', 'โมง'], 'span': [7, 11], 'entity_type': 'time'}, - {'text': ['โมง'], 'span': [9, 11], 'entity_type': 'unit'} + {"text": ["ห้า"], "span": [7, 9], "entity_type": "cardinal"}, + {"text": ["ห้า", "โมง"], "span": [7, 11], "entity_type": "time"}, + {"text": ["โมง"], "span": [9, 11], "entity_type": "unit"}, ] top_entities = get_top_level_entities(entities) # Should only return 'time' as it contains the others self.assertEqual(len(top_entities), 1) - self.assertEqual(top_entities[0]['entity_type'], 'time') - self.assertEqual(top_entities[0]['span'], [7, 11]) + self.assertEqual(top_entities[0]["entity_type"], "time") + self.assertEqual(top_entities[0]["span"], [7, 11]) # Test with non-overlapping entities entities = [ - {'text': ['วัน'], 'span': [0, 1], 'entity_type': 'time'}, - {'text': ['เดือน'], 'span': [2, 3], 'entity_type': 'time'} + {"text": ["วัน"], "span": [0, 1], "entity_type": "time"}, + {"text": ["เดือน"], "span": [2, 3], "entity_type": "time"}, ] top_entities = get_top_level_entities(entities) # Both should be returned as neither contains the other @@ -248,7 +259,7 @@ def test_get_top_level_entities(self): self.assertEqual(get_top_level_entities([]), []) # Test with single entity - entities = [{'text': ['test'], 'span': [0, 1], 'entity_type': 'test'}] + entities = [{"text": ["test"], "span": [0, 1], "entity_type": "test"}] top_entities = get_top_level_entities(entities) self.assertEqual(len(top_entities), 1) self.assertEqual(top_entities[0], entities[0]) @@ -257,41 +268,53 @@ def test_entities_to_iob(self): from pythainlp.tag.thai_nner import _entities_to_iob # Test basic IOB conversion - tokens = ['วัน', 'ที่', ' ', '5', ' ', 'เมษายน'] + tokens = ["วัน", "ที่", " ", "5", " ", "เมษายน"] entities = [ - {'text': ['5', ' ', 'เมษายน'], 'span': [3, 6], 'entity_type': 'date'} + { + "text": ["5", " ", "เมษายน"], + "span": [3, 6], + "entity_type": "date", + } ] result = _entities_to_iob(tokens, entities) # Check format self.assertEqual(len(result), len(tokens)) - self.assertEqual(result[0], ('วัน', 'O')) - self.assertEqual(result[1], ('ที่', 'O')) - self.assertEqual(result[2], (' ', 'O')) - self.assertEqual(result[3], ('5', 'B-DATE')) - self.assertEqual(result[4], (' ', 'I-DATE')) - self.assertEqual(result[5], ('เมษายน', 'I-DATE')) + self.assertEqual(result[0], ("วัน", "O")) + self.assertEqual(result[1], ("ที่", "O")) + self.assertEqual(result[2], (" ", "O")) + self.assertEqual(result[3], ("5", "B-DATE")) + self.assertEqual(result[4], (" ", "I-DATE")) + self.assertEqual(result[5], ("เมษายน", "I-DATE")) def test_entities_to_html(self): from pythainlp.tag.thai_nner import _entities_to_html # Test basic HTML conversion - tokens = ['วัน', 'ที่', ' ', '5', ' ', 'เมษายน'] + tokens = ["วัน", "ที่", " ", "5", " ", "เมษายน"] entities = [ - {'text': ['5', ' ', 'เมษายน'], 'span': [3, 6], 'entity_type': 'date'} + { + "text": ["5", " ", "เมษายน"], + "span": [3, 6], + "entity_type": "date", + } ] result = _entities_to_html(tokens, entities) # Check format - expected = 'วันที่ 5 เมษายน' + expected = "วันที่ 5 เมษายน" self.assertEqual(result, expected) # Test with multiple entities - tokens = ['นาย', 'สมชาย', ' ', 'อยู่', 'ที่', 'กรุงเทพ'] + tokens = ["นาย", "สมชาย", " ", "อยู่", "ที่", "กรุงเทพ"] entities = [ - {'text': ['นาย', 'สมชาย'], 'span': [0, 2], 'entity_type': 'person'}, - {'text': ['กรุงเทพ'], 'span': [5, 6], 'entity_type': 'location'} + { + "text": ["นาย", "สมชาย"], + "span": [0, 2], + "entity_type": "person", + }, + {"text": ["กรุงเทพ"], "span": [5, 6], "entity_type": "location"}, ] result = _entities_to_html(tokens, entities) - expected = 'นายสมชาย อยู่ที่กรุงเทพ' + expected = "นายสมชาย อยู่ที่กรุงเทพ" self.assertEqual(result, expected) diff --git a/tests/core/test_tools.py b/tests/core/test_tools.py index 72c473395..04ec2de95 100644 --- a/tests/core/test_tools.py +++ b/tests/core/test_tools.py @@ -6,12 +6,17 @@ import tempfile import unittest import warnings +from unittest.mock import patch +from pythainlp import is_offline_mode from pythainlp.tools import ( get_full_data_path, get_pythainlp_data_path, get_pythainlp_path, ) +from pythainlp.tools import ( + is_offline_mode as tools_is_offline_mode, +) from pythainlp.tools.core import safe_print, warn_deprecation @@ -24,8 +29,22 @@ def test_path(self): self.assertIsInstance(get_pythainlp_data_path(), str) self.assertIsInstance(get_pythainlp_path(), str) + def test_custom_data_dir_new(self): + """Test that PYTHAINLP_DATA environment variable is respected.""" + with tempfile.TemporaryDirectory() as temp_dir: + custom_dir = os.path.join(temp_dir, "pythainlp-data") + with patch.dict( + os.environ, + {"PYTHAINLP_DATA": custom_dir}, + clear=False, + ): + os.environ.pop("PYTHAINLP_DATA_DIR", None) + path = get_pythainlp_data_path() + self.assertEqual(path, custom_dir) + self.assertTrue(os.path.isdir(path)) + def test_custom_data_dir(self): - """Test that PYTHAINLP_DATA_DIR environment variable is respected. + """Test that PYTHAINLP_DATA_DIR is accepted but emits a deprecation warning. This test verifies the functionality needed for distributed environments like PySpark, where setting PYTHAINLP_DATA_DIR @@ -33,32 +52,67 @@ def test_custom_data_dir(self): See: https://github.com/PyThaiNLP/pythainlp/issues/475 """ - # Save original value - original_value = os.environ.get("PYTHAINLP_DATA_DIR") - - # Use temporary directory for hermetic test with tempfile.TemporaryDirectory() as temp_dir: - try: - # Test with custom directory - custom_dir = os.path.join(temp_dir, "pythainlp-data") - os.environ["PYTHAINLP_DATA_DIR"] = custom_dir - - # Get path should return the custom directory - path = get_pythainlp_data_path() - - # Verify the path matches our custom directory + custom_dir = os.path.join(temp_dir, "pythainlp-data") + with patch.dict( + os.environ, + {"PYTHAINLP_DATA_DIR": custom_dir}, + clear=False, + ): + os.environ.pop("PYTHAINLP_DATA", None) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + path = get_pythainlp_data_path() self.assertEqual(path, custom_dir) - - # Verify directory was created - self.assertTrue(os.path.exists(path)) self.assertTrue(os.path.isdir(path)) + self.assertEqual(len(w), 1) + self.assertTrue(issubclass(w[0].category, DeprecationWarning)) + self.assertIn("PYTHAINLP_DATA_DIR", str(w[0].message)) + self.assertIn("PYTHAINLP_DATA", str(w[0].message)) - finally: - # Restore original value - if original_value is not None: - os.environ["PYTHAINLP_DATA_DIR"] = original_value - elif "PYTHAINLP_DATA_DIR" in os.environ: - del os.environ["PYTHAINLP_DATA_DIR"] + def test_custom_data_dir_conflict(self): + """Test that setting both PYTHAINLP_DATA and PYTHAINLP_DATA_DIR raises ValueError.""" + with tempfile.TemporaryDirectory() as temp_dir: + with patch.dict( + os.environ, + { + "PYTHAINLP_DATA": os.path.join(temp_dir, "a"), + "PYTHAINLP_DATA_DIR": os.path.join(temp_dir, "b"), + }, + ): + with self.assertRaises(ValueError) as ctx: + get_pythainlp_data_path() + self.assertIn("PYTHAINLP_DATA", str(ctx.exception)) + self.assertIn("PYTHAINLP_DATA_DIR", str(ctx.exception)) + + def test_is_offline_mode(self): + """Test is_offline_mode() reflects PYTHAINLP_OFFLINE env var.""" + # Truthy values + for truthy in ("1", "true", "True", "TRUE", "yes", "YES", "on", "ON"): + with patch.dict(os.environ, {"PYTHAINLP_OFFLINE": truthy}): + self.assertTrue( + is_offline_mode(), + f"Expected offline for PYTHAINLP_OFFLINE={truthy!r}", + ) + # Falsy values + for falsy in ( + "", + "0", + "false", + "False", + "FALSE", + "no", + "NO", + "off", + "OFF", + ): + with patch.dict(os.environ, {"PYTHAINLP_OFFLINE": falsy}): + self.assertFalse( + is_offline_mode(), + f"Expected online for PYTHAINLP_OFFLINE={falsy!r}", + ) + # Same function is exposed via pythainlp.tools + self.assertIs(is_offline_mode, tools_is_offline_mode) def test_warn_deprecation(self): """Test deprecation warning function.""" @@ -83,9 +137,7 @@ def test_warn_deprecation(self): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") warn_deprecation( - "old_func", - deprecated_version="1.0", - removal_version="2.0" + "old_func", deprecated_version="1.0", removal_version="2.0" ) self.assertEqual(len(w), 1) self.assertIn("1.0", str(w[0].message)) @@ -98,7 +150,7 @@ def test_warn_deprecation(self): "old_func", replacing_func="new_func", deprecated_version="1.0", - removal_version="2.0" + removal_version="2.0", ) self.assertEqual(len(w), 1) message = str(w[0].message)