diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index 211020a87..47f2f774b 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -11,7 +11,7 @@ __all__: list[str] = ["gen_sentence"] -import pickle +import json import random from typing import TYPE_CHECKING, Any, Union @@ -58,22 +58,18 @@ thwiki: dict[str, Any] = THWIKI_LSTM # Validate that corpus files are available -if thwiki["itos_fname"] is None or thwiki["wgts_fname"] is None: +if thwiki["json_itos_fname"] is None or thwiki["wgts_fname"] is None: raise FileNotFoundError( "corpus-not-found names=['wiki_lm_lstm', 'wiki_itos_lstm']\n" " Thai2fit model files not found.\n" " Python: pythainlp.corpus.download('wiki_lm_lstm')\n" " CLI: thainlp data get wiki_lm_lstm\n" - " Python: pythainlp.corpus.download('wiki_itos_lstm')\n" - " CLI: thainlp data get wiki_itos_lstm" + " Python: pythainlp.corpus.download('json_itos_fname')\n" + " CLI: thainlp data get json_itos_fname" ) -# Loads a pickle file from PyThaiNLP's official repository with MD5 verification. -# WARNING: -# Pickle deserialization can execute arbitrary code if the file is malicious. -# Users should only use files from trusted sources. -with open(thwiki["itos_fname"], "rb") as f: - thwiki_itos: list[str] = pickle.load(f) # noqa: S301 +with open(thwiki["json_itos_fname"], "r") as f: + thwiki_itos: list[str] = json.load(f) thwiki_vocab: "Vocab" = fastai.text.transform.Vocab(thwiki_itos) # dummy databunch diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index ae2e29fad..21f60a091 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -51,6 +51,7 @@ THWIKI_LSTM: dict[str, Optional[str]] = { "wgts_fname": get_corpus_path(_MODEL_NAME_LSTM), "itos_fname": get_corpus_path(_ITOS_NAME_LSTM), + "json_itos_fname": get_corpus_path("wiki_itos_lstm_json"), }