From cf533a633a4675c74f94c3f125e1d30296f7b8ed Mon Sep 17 00:00:00 2001 From: Wannaphong Phatthiyaphaibun Date: Fri, 13 Mar 2026 21:56:39 +0700 Subject: [PATCH 1/4] Add json_itos_fname to THWIKI_LSTM dictionary --- pythainlp/ulmfit/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index ae2e29fad..21f60a091 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -51,6 +51,7 @@ THWIKI_LSTM: dict[str, Optional[str]] = { "wgts_fname": get_corpus_path(_MODEL_NAME_LSTM), "itos_fname": get_corpus_path(_ITOS_NAME_LSTM), + "json_itos_fname": get_corpus_path("wiki_itos_lstm_json"), } From 06a8ee898ac86581b30a2eac78096e234325d593 Mon Sep 17 00:00:00 2001 From: Wannaphong Phatthiyaphaibun Date: Fri, 13 Mar 2026 21:57:57 +0700 Subject: [PATCH 2/4] Replace pickle with JSON for loading vocabulary --- pythainlp/generate/thai2fit.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index 211020a87..9ee024b2f 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -11,7 +11,7 @@ __all__: list[str] = ["gen_sentence"] -import pickle +import json import random from typing import TYPE_CHECKING, Any, Union @@ -58,22 +58,22 @@ thwiki: dict[str, Any] = THWIKI_LSTM # Validate that corpus files are available -if thwiki["itos_fname"] is None or thwiki["wgts_fname"] is None: +if thwiki["json_itos_fname"] is None or thwiki["wgts_fname"] is None: raise FileNotFoundError( "corpus-not-found names=['wiki_lm_lstm', 'wiki_itos_lstm']\n" " Thai2fit model files not found.\n" " Python: pythainlp.corpus.download('wiki_lm_lstm')\n" " CLI: thainlp data get wiki_lm_lstm\n" - " Python: pythainlp.corpus.download('wiki_itos_lstm')\n" - " CLI: thainlp data get wiki_itos_lstm" + " Python: pythainlp.corpus.download('json_itos_fname')\n" + " CLI: thainlp data get json_itos_fname" ) # Loads a pickle file from PyThaiNLP's official repository with MD5 verification. # WARNING: # Pickle deserialization can execute arbitrary code if the file is malicious. # Users should only use files from trusted sources. -with open(thwiki["itos_fname"], "rb") as f: - thwiki_itos: list[str] = pickle.load(f) # noqa: S301 +with open(thwiki["json_itos_fname"], "r") as f: + thwiki_itos: list[str] = json.load(f) # noqa: S301 thwiki_vocab: "Vocab" = fastai.text.transform.Vocab(thwiki_itos) # dummy databunch From 55f76ba0e022d9e55b0515566ea394c30166f5db Mon Sep 17 00:00:00 2001 From: Wannaphong Phatthiyaphaibun Date: Sat, 14 Mar 2026 00:14:57 +0700 Subject: [PATCH 3/4] Update pythainlp/generate/thai2fit.py Co-authored-by: Arthit Suriyawongkul --- pythainlp/generate/thai2fit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index 9ee024b2f..a30178768 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -73,7 +73,7 @@ # Pickle deserialization can execute arbitrary code if the file is malicious. # Users should only use files from trusted sources. with open(thwiki["json_itos_fname"], "r") as f: - thwiki_itos: list[str] = json.load(f) # noqa: S301 + thwiki_itos: list[str] = json.load(f) thwiki_vocab: "Vocab" = fastai.text.transform.Vocab(thwiki_itos) # dummy databunch From cb18b9a0fbec2da72289f709a43677cf24697fe4 Mon Sep 17 00:00:00 2001 From: Wannaphong Phatthiyaphaibun Date: Sat, 14 Mar 2026 00:15:06 +0700 Subject: [PATCH 4/4] Update pythainlp/generate/thai2fit.py Co-authored-by: Arthit Suriyawongkul --- pythainlp/generate/thai2fit.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index a30178768..47f2f774b 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -68,10 +68,6 @@ " CLI: thainlp data get json_itos_fname" ) -# Loads a pickle file from PyThaiNLP's official repository with MD5 verification. -# WARNING: -# Pickle deserialization can execute arbitrary code if the file is malicious. -# Users should only use files from trusted sources. with open(thwiki["json_itos_fname"], "r") as f: thwiki_itos: list[str] = json.load(f) thwiki_vocab: "Vocab" = fastai.text.transform.Vocab(thwiki_itos)