From 41ff37be9a2d225a2032dd10377e37003998ea58 Mon Sep 17 00:00:00 2001 From: Wannaphong Phatthiyaphaibun Date: Fri, 17 Oct 2025 14:29:11 +0700 Subject: [PATCH 1/5] Add get_hf_hub and make_safe_directory_name --- pythainlp/corpus/__init__.py | 4 +++ pythainlp/corpus/core.py | 50 ++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/pythainlp/corpus/__init__.py b/pythainlp/corpus/__init__.py index 486c6dcf8..00950da29 100644 --- a/pythainlp/corpus/__init__.py +++ b/pythainlp/corpus/__init__.py @@ -23,6 +23,7 @@ "get_corpus_default_db", "get_corpus_path", "get_path_folder_corpus", + "get_hf_hub", "path_pythainlp_corpus", "provinces", "remove", @@ -41,6 +42,7 @@ "thai_wikipedia_titles", "thai_words", "thai_wsd_dict", + "make_safe_directory_name", ] import os @@ -98,6 +100,8 @@ def corpus_db_path() -> str: get_corpus_default_db, get_corpus_path, get_path_folder_corpus, + make_safe_directory_name, + get_hf_hub, path_pythainlp_corpus, remove, ) # these imports must come before other pythainlp.corpus.* imports diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index c3eb31c0c..997ccb8f1 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -8,6 +8,7 @@ import json import os +import re from typing import Union from pythainlp import __version__ @@ -614,3 +615,52 @@ def remove(name: str) -> bool: def get_path_folder_corpus(name, version, *path): return os.path.join(get_corpus_path(name, version), *path) + + +def make_safe_directory_name(name:str) -> str: + """ + Make safe directory name + + :param str name: directory name + :return: safe directory name + :rtype: str + """ + # Replace invalid characters with an underscore + safe_name = re.sub(r'[<>:"/\\|?*]', '_', name) + # Remove leading/trailing spaces or periods (especially important for Windows) + safe_name = safe_name.strip(' .') + # Prevent names that are reserved on Windows + reserved_names = ['CON', 'PRN', 'AUX', 'NUL', 'COM1', 'COM2', 'COM3', 'COM4', 'COM5', 'COM6', 'COM7', 'COM8', 'COM9', 'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5', 'LPT6', 'LPT7', 'LPT8', 'LPT9'] + if safe_name.upper() in reserved_names: + safe_name = f"_{safe_name}" # Prepend underscore to avoid conflict + return safe_name + + +def get_hf_hub(repo_id:str, filename: str=None) -> str: + """ + HuggingFace Hub in pythainlp-home + + :param str repo_id: repo_id + :param str filename: filename + :return: path + :rtype: str + """ + if _CHECK_MODE == "1": + print("PyThaiNLP is read-only mode. It can't download.") + return False + from huggingface_hub import hf_hub_download, snapshot_download + hf_root = get_full_data_path("hf_models") + name_dir = make_safe_directory_name(repo_id) + root_project = os.path.join(hf_root, name_dir) + if filename!=None: + output_path = hf_hub_download( + repo_id=repo_id, + filename=filename, + local_dir=root_project + ) + else: + output_path = snapshot_download( + repo_id=repo_id, + local_dir=root_project + ) + return output_path From 39a6136be63a2d89d8ee8bbaa151fad3d7d925a4 Mon Sep 17 00:00:00 2001 From: Wannaphong Phatthiyaphaibun Date: Fri, 17 Oct 2025 14:34:36 +0700 Subject: [PATCH 2/5] Update core.py --- pythainlp/corpus/core.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 997ccb8f1..6eca675c1 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -648,7 +648,13 @@ def get_hf_hub(repo_id:str, filename: str=None) -> str: if _CHECK_MODE == "1": print("PyThaiNLP is read-only mode. It can't download.") return False - from huggingface_hub import hf_hub_download, snapshot_download + try: + from huggingface_hub import hf_hub_download, snapshot_download + except: + raise ModuleNotFoundError(""" + huggingface-hub isn't found! + Please installing the package via 'pip install huggingface-hub'. + """) hf_root = get_full_data_path("hf_models") name_dir = make_safe_directory_name(repo_id) root_project = os.path.join(hf_root, name_dir) From 0e960c919002abfbe7e0a952b90822e24309897d Mon Sep 17 00:00:00 2001 From: Wannaphong Phatthiyaphaibun Date: Fri, 17 Oct 2025 14:39:33 +0700 Subject: [PATCH 3/5] Update core.py --- pythainlp/corpus/core.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 6eca675c1..addaa9968 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -585,9 +585,6 @@ def remove(name: str) -> bool: # FileNotFoundError: [Errno 2] No such file or directory: # '/usr/local/lib/python3.6/dist-packages/pythainlp/corpus/ttc' """ - if _CHECK_MODE == "1": - print("PyThaiNLP is read-only mode. It can't remove corpus.") - return False with open(corpus_db_path(), "r", encoding="utf-8-sig") as f: db = json.load(f) data = [ From 24349791726ee505bb7edd42a4feccd3b8e60661 Mon Sep 17 00:00:00 2001 From: Wannaphong Phatthiyaphaibun Date: Fri, 17 Oct 2025 14:47:54 +0700 Subject: [PATCH 4/5] Update core.py --- pythainlp/corpus/core.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index addaa9968..4079edacd 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -647,11 +647,13 @@ def get_hf_hub(repo_id:str, filename: str=None) -> str: return False try: from huggingface_hub import hf_hub_download, snapshot_download - except: + except ModuleNotFoundError: raise ModuleNotFoundError(""" huggingface-hub isn't found! Please installing the package via 'pip install huggingface-hub'. """) + except Exception as e: + raise Exception(f"An unexpected error occurred: {e}") hf_root = get_full_data_path("hf_models") name_dir = make_safe_directory_name(repo_id) root_project = os.path.join(hf_root, name_dir) From 34802457da67aab98926fce2492dfd9f6393ad6c Mon Sep 17 00:00:00 2001 From: Wannaphong Phatthiyaphaibun Date: Fri, 17 Oct 2025 14:52:01 +0700 Subject: [PATCH 5/5] Update core.py --- pythainlp/corpus/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 4079edacd..9df9fdfc3 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -635,7 +635,7 @@ def make_safe_directory_name(name:str) -> str: def get_hf_hub(repo_id:str, filename: str=None) -> str: """ - HuggingFace Hub in pythainlp-home + HuggingFace Hub in :mod:`pythainlp` data directory. :param str repo_id: repo_id :param str filename: filename