diff --git a/build_tools/analysis/output/functions_incomplete_hints.csv b/build_tools/analysis/output/functions_incomplete_hints.csv index 252bb771e..26d33430d 100644 --- a/build_tools/analysis/output/functions_incomplete_hints.csv +++ b/build_tools/analysis/output/functions_incomplete_hints.csv @@ -1,57 +1 @@ Function Name,Submodule,Scope,Priority,Params Hinted,Has Return,References,Test Suite,File,Line -pythainlp.tokenize.nercut.segment,tokenize,public,medium,2/3,True,350,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/nercut.py,22 -pythainlp.classify.param_free.GzipModel.load,classify,public,medium,1/1,False,334,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/classify/param_free.py,101 -pythainlp.transliterate.thaig2p_v2.transliterate,transliterate,public,medium,1/2,True,127,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p_v2.py,41 -pythainlp.transliterate.umt5_thaig2p.transliterate,transliterate,public,medium,1/2,True,127,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/umt5_thaig2p.py,41 -pythainlp.transliterate.wunsen.WunsenTransliterate.transliterate,transliterate,public,medium,5/5,False,127,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py,37 -pythainlp.augment.lm.fasttext.FastTextAug.__init__,augment,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/fasttext.py,17 -pythainlp.augment.word2vec.bpemb_wv.BPEmbAug.__init__,augment,public,medium,3/3,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/bpemb_wv.py,16 -pythainlp.classify.param_free.GzipModel.__init__,classify,public,medium,2/2,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/classify/param_free.py,24 -pythainlp.coref._fastcoref.FastCoref.__init__,coref,public,medium,2/4,True,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/coref/_fastcoref.py,8 -pythainlp.coref.han_coref.HanCoref.__init__,coref,public,medium,1/2,True,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/coref/han_coref.py,10 -pythainlp.el.core.EntityLinker.__init__,el,public,medium,3/3,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/el/core.py,10 -pythainlp.generate.core.Unigram.__init__,generate,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/generate/core.py,33 -pythainlp.generate.core.Bigram.__init__,generate,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/generate/core.py,119 -pythainlp.generate.core.Trigram.__init__,generate,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/generate/core.py,206 -pythainlp.spell.pn.NorvigSpellChecker.__init__,spell,public,medium,5/5,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/spell/pn.py,123 -pythainlp.summarize.freq.FrequencySummarizer.__init__,summarize,public,medium,2/2,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/summarize/freq.py,20 -pythainlp.summarize.keybert.KeyBERT.__init__,summarize,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/summarize/keybert.py,27 -pythainlp.summarize.mt5.mT5Summarizer.__init__,summarize,public,medium,7/7,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/summarize/mt5.py,12 -pythainlp.tag.crfchunk.CRFchunk.__init__,tag,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tag/crfchunk.py,77 -pythainlp.tokenize.core.Tokenizer.__init__,tokenize,public,medium,4/4,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/core.py,925 -pythainlp.tokenize.longest.LongestMatchTokenizer.__init__,tokenize,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/longest.py,49 -pythainlp.translate.en_th.EnThTranslator.__init__,translate,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/translate/en_th.py,68 -pythainlp.translate.en_th.ThEnTranslator.__init__,translate,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/translate/en_th.py,124 -pythainlp.translate.tokenization_small100.SMALL100Tokenizer.__init__,translate,public,medium,1/11,True,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py,125 -pythainlp.transliterate.thaig2p_v2.ThaiG2P.__init__,transliterate,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p_v2.py,27 -pythainlp.transliterate.umt5_thaig2p.Umt5ThaiG2P.__init__,transliterate,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/umt5_thaig2p.py,27 -pythainlp.ulmfit.tokenizer.BaseTokenizer.__init__,ulmfit,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/tokenizer.py,16 -pythainlp.ulmfit.tokenizer.ThaiTokenizer.__init__,ulmfit,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/tokenizer.py,32 -pythainlp.util.trie.Trie.__init__,util,public,medium,1/1,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/util/trie.py,56 -pythainlp.wangchanberta.core.ThaiNameTagger.__init__,wangchanberta,public,medium,2/2,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py,35 -pythainlp.wsd.core._SentenceTransformersModel.__init__,wsd,public,medium,2/2,False,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py,26 -pythainlp.augment.lm.wangchanberta.Thai2transformersAug.generate,augment,public,medium,2/2,False,37,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/wangchanberta.py,34 -pythainlp.classify.param_free.GzipModel.save,classify,public,medium,1/1,False,37,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/classify/param_free.py,89 -pythainlp.util.date.convert_years,util,public,medium,1/3,True,24,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py,119 -pythainlp.summarize.keybert.KeyBERT.extract_keywords,summarize,public,medium,6/7,True,20,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/summarize/keybert.py,39 -pythainlp.spell.symspellpy.correct_sent,spell,public,medium,1/2,True,19,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/spell/symspellpy.py,111 -pythainlp.chat.core.ChatBotModel.load_model,chat,public,medium,6/7,False,18,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/chat/core.py,18 -pythainlp.generate.wangchanglm.WangChanGLM.load_model,generate,public,medium,6/7,False,18,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/generate/wangchanglm.py,26 -pythainlp.util.emojiconv.emoji_to_thai,util,public,medium,1/2,True,15,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/util/emojiconv.py,1835 -pythainlp.util.date.thai_strptime,util,public,medium,4/5,False,12,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py,212 -pythainlp.ulmfit.core.document_vector,ulmfit,public,medium,2/4,False,11,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/core.py,175 -pythainlp.augment.wordnet.postype2wordnet,augment,public,medium,2/2,False,6,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/augment/wordnet.py,104 -pythainlp.ulmfit.tokenizer.BaseTokenizer.add_special_cases,ulmfit,public,medium,1/1,False,4,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/tokenizer.py,22 -pythainlp.transliterate.thai2rom.ThaiTransliterator._prepare_sequence_in,transliterate,private,low,1/1,False,12,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py,61 -pythainlp.transliterate.thai2rom_onnx.ThaiTransliterator_ONNX._prepare_sequence_in,transliterate,private,low,1/1,False,12,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py,59 -pythainlp.transliterate.thaig2p.ThaiG2P._prepare_sequence_in,transliterate,private,low,1/1,False,12,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py,71 -pythainlp.benchmarks.word_tokenization._find_word_boundaries,benchmarks,private,low,0/1,True,5,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/benchmarks/word_tokenization.py,239 -pythainlp.transliterate.w2p.Thai_W2P._gru,transliterate,private,low,0/7,True,5,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py,121 -pythainlp.tag.thainer._doc2features,tag,private,low,0/2,True,4,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tag/thainer.py,25 -pythainlp.generate.wangchanglm.WangChanGLM.gen_instruct,generate,public,low,9/9,False,3,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/generate/wangchanglm.py,68 -pythainlp.corpus.wordnet.custom_lemmas,corpus,public,low,1/2,True,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/wordnet.py,437 -pythainlp.generate.wangchanglm.WangChanGLM.instruct_generate,generate,public,low,9/10,False,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/generate/wangchanglm.py,124 -pythainlp.summarize.freq.FrequencySummarizer.__rank,summarize,private,low,1/2,False,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/summarize/freq.py,26 -pythainlp.translate.tokenization_small100.save_json,translate,public,low,1/2,True,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py,433 -pythainlp.wsd.core._SentenceTransformersModel.change_device,wsd,public,low,1/1,False,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py,37 -pythainlp.translate.tokenization_small100.SMALL100Tokenizer._build_translation_inputs,translate,private,low,1/2,False,1,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py,384 diff --git a/build_tools/analysis/output/functions_no_hints.csv b/build_tools/analysis/output/functions_no_hints.csv index 6c0a25cb9..4b3e89e07 100644 --- a/build_tools/analysis/output/functions_no_hints.csv +++ b/build_tools/analysis/output/functions_no_hints.csv @@ -1,73 +1 @@ Function Name,Submodule,Scope,Priority,References,Test Suite,File,Line -pythainlp.corpus.util.tokenize,corpus,public,medium,991,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/util.py,136 -pythainlp.classify.param_free.GzipModel.train,classify,public,medium,164,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/classify/param_free.py,35 -pythainlp.augment.lm.wangchanberta.Thai2transformersAug.__init__,augment,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/wangchanberta.py,10 -pythainlp.augment.word2vec.ltw2v.LTW2VAug.__init__,augment,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/ltw2v.py,18 -pythainlp.augment.word2vec.thai2fit.Thai2fitAug.__init__,augment,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/thai2fit.py,18 -pythainlp.augment.wordnet.WordNetAug.__init__,augment,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/augment/wordnet.py,121 -pythainlp.chat.core.ChatBotModel.__init__,chat,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/chat/core.py,10 -pythainlp.el._multiel.MultiEL.__init__,el,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/el/_multiel.py,7 -pythainlp.generate.wangchanglm.WangChanGLM.__init__,generate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/generate/wangchanglm.py,12 -pythainlp.khavee.core.KhaveeVerifier.__init__,khavee,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/khavee/core.py,15 -pythainlp.phayathaibert.core.ThaiTextProcessor.__init__,phayathaibert,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/phayathaibert/core.py,25 -pythainlp.soundex.complete_soundex.CompleteSoundex.__init__,soundex,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/soundex/complete_soundex.py,46 -pythainlp.tokenize.attacut.AttacutTokenizer.__init__,tokenize,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/attacut.py,19 -pythainlp.tokenize.multi_cut.LatticeString.__init__,tokenize,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/multi_cut.py,31 -pythainlp.transliterate.thai2rom.ThaiTransliterator.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py,22 -pythainlp.transliterate.thai2rom.Encoder.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py,101 -pythainlp.transliterate.thai2rom.Attn.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py,165 -pythainlp.transliterate.thai2rom.AttentionDecoder.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py,211 -pythainlp.transliterate.thai2rom.Seq2Seq.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py,261 -pythainlp.transliterate.thai2rom_onnx.ThaiTransliterator_ONNX.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py,21 -pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py,91 -pythainlp.transliterate.thaig2p.ThaiG2P.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py,36 -pythainlp.transliterate.thaig2p.Encoder.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py,112 -pythainlp.transliterate.thaig2p.Attn.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py,178 -pythainlp.transliterate.thaig2p.AttentionDecoder.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py,224 -pythainlp.transliterate.thaig2p.Seq2Seq.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py,274 -pythainlp.transliterate.w2p.Thai_W2P.__init__,transliterate,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py,52 -pythainlp.util.trie.Node.__init__,util,public,medium,113,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/util/trie.py,52 -pythainlp.__main__.main,__main__,public,medium,58,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/__main__.py,10 -pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.run,transliterate,public,medium,53,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py,115 -pythainlp.el._multiel.MultiEL.load_model,el,public,medium,18,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/el/_multiel.py,12 -pythainlp.transliterate.thai2rom.Encoder.forward,transliterate,public,medium,15,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py,119 -pythainlp.transliterate.thai2rom.Attn.forward,transliterate,public,medium,15,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py,178 -pythainlp.transliterate.thai2rom.AttentionDecoder.forward,transliterate,public,medium,15,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py,233 -pythainlp.transliterate.thai2rom.Seq2Seq.forward,transliterate,public,medium,15,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py,288 -pythainlp.transliterate.thaig2p.Encoder.forward,transliterate,public,medium,15,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py,130 -pythainlp.transliterate.thaig2p.Attn.forward,transliterate,public,medium,15,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py,191 -pythainlp.transliterate.thaig2p.AttentionDecoder.forward,transliterate,public,medium,15,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py,246 -pythainlp.transliterate.thaig2p.Seq2Seq.forward,transliterate,public,medium,15,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py,301 -pythainlp.tokenize.thai2fit.thai2fit_tokenizer,tokenize,public,medium,12,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/thai2fit.py,12 -pythainlp.ulmfit.core.merge_wgts,ulmfit,public,medium,9,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/core.py,230 -pythainlp.augment.word2vec.bpemb_wv.BPEmbAug.load_w2v,augment,public,medium,6,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/bpemb_wv.py,29 -pythainlp.augment.word2vec.ltw2v.LTW2VAug.load_w2v,augment,public,medium,6,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/ltw2v.py,28 -pythainlp.augment.word2vec.thai2fit.Thai2fitAug.load_w2v,augment,public,medium,6,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/thai2fit.py,29 -pythainlp.transliterate.thai2rom.Seq2Seq.create_mask,transliterate,public,medium,6,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py,284 -pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.create_mask,transliterate,public,medium,6,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py,111 -pythainlp.transliterate.thaig2p.Seq2Seq.create_mask,transliterate,public,medium,6,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py,297 -pythainlp.tokenize.multi_cut.serialize,tokenize,public,medium,4,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/multi_cut.py,62 -pythainlp.transliterate.thai2rom.Encoder.init_hidden,transliterate,public,medium,4,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py,153 -pythainlp.transliterate.thaig2p.Encoder.init_hidden,transliterate,public,medium,4,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py,166 -pythainlp.ulmfit.tokenizer.ThaiTokenizer.add_special_cases,ulmfit,public,medium,4,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/tokenizer.py,67 -pythainlp.phayathaibert.core.ThaiTextProcessor._replace_rep,phayathaibert,private,low,8,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/phayathaibert/core.py,129 -pythainlp.ulmfit.preprocess._replace_rep,ulmfit,private,low,8,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/preprocess.py,104 -pythainlp.ulmfit.preprocess._replace_rep,ulmfit,private,low,8,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/preprocess.py,227 -pythainlp.util.normalize._last_char,util,private,low,7,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/util/normalize.py,63 -pythainlp.el._multiel.MultiEL.process_batch,el,public,low,3,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/el/_multiel.py,21 -pythainlp.transliterate.w2p.Thai_W2P._grucell,transliterate,private,low,3,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py,100 -pythainlp.wangchanberta.core._get_tokenizer,wangchanberta,private,low,3,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py,16 -pythainlp.wangchanberta.core.ThaiNameTagger._clear_tag,wangchanberta,private,low,3,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py,65 -pythainlp.coref._fastcoref.FastCoref._to_json,coref,private,low,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/coref/_fastcoref.py,29 -pythainlp.tokenize.budoux._init_parser,tokenize,private,low,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/budoux.py,22 -pythainlp.tokenize.etcc._cut_etcc,tokenize,private,low,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/etcc.py,32 -pythainlp.tokenize.multi_cut.LatticeString.__new__,tokenize,public,low,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/multi_cut.py,28 -pythainlp.tokenize.nlpo3._ensure_default_dict_loaded,tokenize,private,low,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/nlpo3.py,23 -pythainlp.transliterate.w2p._load_vocab,transliterate,private,low,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py,41 -pythainlp.transliterate.w2p.Thai_W2P._sigmoid,transliterate,private,low,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py,97 -pythainlp.wangchanberta.core.ThaiNameTagger._IOB,wangchanberta,private,low,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py,60 -pythainlp.wangchanberta.core.NamedEntityRecognition._fix_span_error,wangchanberta,private,low,2,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py,161 -pythainlp.chat.core.ChatBotModel.reset_chat,chat,public,low,1,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/chat/core.py,14 -pythainlp.khavee.core.KhaveeVerifier.check_karu_lahu,khavee,public,low,1,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/khavee/core.py,359 -pythainlp.translate.tokenization_small100.SMALL100Tokenizer._switch_to_input_mode,translate,private,low,1,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py,397 -pythainlp.translate.tokenization_small100.SMALL100Tokenizer._switch_to_target_mode,translate,private,low,1,unknown,/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py,400 diff --git a/build_tools/analysis/output/submodule_summary.csv b/build_tools/analysis/output/submodule_summary.csv index ebe12713a..c56f26f31 100644 --- a/build_tools/analysis/output/submodule_summary.csv +++ b/build_tools/analysis/output/submodule_summary.csv @@ -1,32 +1,32 @@ Submodule,Total,Complete,Incomplete,None,% Complete,Mypy Errors -__main__,1,0,0,1,0.00%,0 -ancient,2,2,0,0,100.00%,0 -augment,29,18,4,7,62.07%,0 -benchmarks,8,7,1,0,87.50%,0 -chat,4,1,1,2,25.00%,0 -classify,5,1,3,1,20.00%,0 -cli,21,21,0,0,100.00%,0 -coref,5,2,2,1,40.00%,0 -corpus,70,68,1,1,97.14%,0 -el,5,1,1,3,20.00%,0 -generate,15,8,6,1,53.33%,0 -khavee,9,7,0,2,77.78%,0 -lm,2,2,0,0,100.00%,0 -morpheme,2,2,0,0,100.00%,0 -parse,9,9,0,0,100.00%,0 -phayathaibert,19,17,0,2,89.47%,0 -soundex,27,26,0,1,96.30%,0 -spell,43,41,2,0,95.35%,0 -summarize,17,12,5,0,70.59%,0 -tag,68,66,2,0,97.06%,0 -tokenize,73,62,3,8,84.93%,0 +__main__,1,1,0,0,100.00%,0 +ancient,2,2,0,0,100.00%,1 +augment,29,29,0,0,100.00%,1 +benchmarks,8,8,0,0,100.00%,1 +chat,4,4,0,0,100.00%,1 +classify,5,5,0,0,100.00%,1 +cli,21,21,0,0,100.00%,1 +coref,5,5,0,0,100.00%,1 +corpus,70,70,0,0,100.00%,1 +el,5,5,0,0,100.00%,1 +generate,15,15,0,0,100.00%,1 +khavee,9,9,0,0,100.00%,1 +lm,2,2,0,0,100.00%,1 +morpheme,2,2,0,0,100.00%,1 +parse,9,9,0,0,100.00%,1 +phayathaibert,19,19,0,0,100.00%,1 +soundex,27,27,0,0,100.00%,1 +spell,43,43,0,0,100.00%,1 +summarize,17,17,0,0,100.00%,1 +tag,68,68,0,0,100.00%,1 +tokenize,73,73,0,0,100.00%,1 tokenizeicu,3,3,0,0,100.00%,0 -tools,9,9,0,0,100.00%,0 -translate,44,37,5,2,84.09%,0 -transliterate,75,36,9,30,48.00%,0 +tools,9,9,0,0,100.00%,1 +translate,44,44,0,0,100.00%,1 +transliterate,75,75,0,0,100.00%,1 transliterateicu,1,1,0,0,100.00%,0 -ulmfit,25,17,4,4,68.00%,0 -util,109,103,4,2,94.50%,0 -wangchanberta,9,4,1,4,44.44%,0 -word_vector,7,7,0,0,100.00%,0 -wsd,4,2,2,0,50.00%,0 +ulmfit,25,25,0,0,100.00%,1 +util,109,109,0,0,100.00%,1 +wangchanberta,9,9,0,0,100.00%,1 +word_vector,7,7,0,0,100.00%,1 +wsd,4,4,0,0,100.00%,1 diff --git a/build_tools/analysis/output/type_hint_analysis.json b/build_tools/analysis/output/type_hint_analysis.json index 510dbfdf3..1a65f2521 100644 --- a/build_tools/analysis/output/type_hint_analysis.json +++ b/build_tools/analysis/output/type_hint_analysis.json @@ -1,18 +1,36 @@ { "statistics": { - "total": 720, - "complete": 592, - "incomplete": 56, - "none": 72, - "pct_complete": 82.22222222222221, - "pct_incomplete": 7.777777777777778, - "pct_none": 10.0 + "functions": { + "total": 725, + "complete": 725, + "incomplete": 0, + "none": 0, + "pct_complete": 100.0, + "pct_incomplete": 0.0, + "pct_none": 0.0 + }, + "variables": { + "total": 1158, + "complete": 655, + "none": 503, + "pct_complete": 56.56303972366149, + "pct_none": 43.43696027633852, + "class_variables": 205, + "instance_variables": 442, + "module_variables": 511 + }, + "type_aliases": { + "total": 0 + }, + "classes": { + "total": 96 + } }, "by_submodule": { "__main__": { - "complete": 0, + "complete": 1, "incomplete": 0, - "none": 1, + "none": 0, "mypy_errors": 0 }, "ancient": { @@ -22,27 +40,27 @@ "mypy_errors": 0 }, "augment": { - "complete": 18, - "incomplete": 4, - "none": 7, + "complete": 29, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "benchmarks": { - "complete": 7, - "incomplete": 1, + "complete": 8, + "incomplete": 0, "none": 0, "mypy_errors": 0 }, "chat": { - "complete": 1, - "incomplete": 1, - "none": 2, + "complete": 4, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "classify": { - "complete": 1, - "incomplete": 3, - "none": 1, + "complete": 5, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "cli": { @@ -52,33 +70,33 @@ "mypy_errors": 0 }, "coref": { - "complete": 2, - "incomplete": 2, - "none": 1, + "complete": 5, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "corpus": { - "complete": 68, - "incomplete": 1, - "none": 1, + "complete": 70, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "el": { - "complete": 1, - "incomplete": 1, - "none": 3, + "complete": 5, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "generate": { - "complete": 8, - "incomplete": 6, - "none": 1, + "complete": 15, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "khavee": { - "complete": 7, + "complete": 9, "incomplete": 0, - "none": 2, + "none": 0, "mypy_errors": 0 }, "lm": { @@ -100,39 +118,39 @@ "mypy_errors": 0 }, "phayathaibert": { - "complete": 17, + "complete": 19, "incomplete": 0, - "none": 2, + "none": 0, "mypy_errors": 0 }, "soundex": { - "complete": 26, + "complete": 27, "incomplete": 0, - "none": 1, + "none": 0, "mypy_errors": 0 }, "spell": { - "complete": 41, - "incomplete": 2, + "complete": 43, + "incomplete": 0, "none": 0, "mypy_errors": 0 }, "summarize": { - "complete": 12, - "incomplete": 5, + "complete": 17, + "incomplete": 0, "none": 0, "mypy_errors": 0 }, "tag": { - "complete": 66, - "incomplete": 2, + "complete": 73, + "incomplete": 0, "none": 0, "mypy_errors": 0 }, "tokenize": { - "complete": 62, - "incomplete": 3, - "none": 8, + "complete": 73, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "tokenizeicu": { @@ -148,15 +166,15 @@ "mypy_errors": 0 }, "translate": { - "complete": 37, - "incomplete": 5, - "none": 2, + "complete": 44, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "transliterate": { - "complete": 36, - "incomplete": 9, - "none": 30, + "complete": 75, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "transliterateicu": { @@ -166,21 +184,21 @@ "mypy_errors": 0 }, "ulmfit": { - "complete": 17, - "incomplete": 4, - "none": 4, + "complete": 25, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "util": { - "complete": 103, - "incomplete": 4, - "none": 2, + "complete": 109, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "wangchanberta": { - "complete": 4, - "incomplete": 1, - "none": 4, + "complete": 9, + "incomplete": 0, + "none": 0, "mypy_errors": 0 }, "word_vector": { @@ -190,1278 +208,3245 @@ "mypy_errors": 0 }, "wsd": { - "complete": 2, - "incomplete": 2, + "complete": 4, + "incomplete": 0, "none": 0, "mypy_errors": 0 } }, - "functions_no_hints": [ + "functions_no_hints": [], + "functions_incomplete_hints": [], + "class_variables_no_hints": [ { - "name": "pythainlp.corpus.util.tokenize", + "name": "pythainlp.util.trie.Node.__slots__", "scope": "public", - "references": 991, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/util.py", - "line": 136 + "parent_class": "Node", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/trie.py", + "line": 53 + } + ], + "instance_variables_no_hints": [ + { + "name": "pythainlp.augment.lm.fasttext.FastTextAug.model", + "scope": "public", + "parent_class": "FastTextAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/fasttext.py", + "line": 31 }, { - "name": "pythainlp.classify.param_free.GzipModel.train", + "name": "pythainlp.augment.lm.fasttext.FastTextAug.model", "scope": "public", - "references": 164, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/classify/param_free.py", - "line": 35 + "parent_class": "FastTextAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/fasttext.py", + "line": 33 }, { - "name": "pythainlp.augment.lm.wangchanberta.Thai2transformersAug.__init__", + "name": "pythainlp.augment.lm.phayathaibert.ThaiTextAugmenter.tokenizer", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/wangchanberta.py", - "line": 10 + "parent_class": "ThaiTextAugmenter", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/phayathaibert.py", + "line": 31 }, { - "name": "pythainlp.augment.word2vec.ltw2v.LTW2VAug.__init__", + "name": "pythainlp.augment.lm.phayathaibert.ThaiTextAugmenter.model_for_masked_lm", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/ltw2v.py", - "line": 18 + "parent_class": "ThaiTextAugmenter", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/phayathaibert.py", + "line": 32 }, { - "name": "pythainlp.augment.word2vec.thai2fit.Thai2fitAug.__init__", + "name": "pythainlp.augment.lm.phayathaibert.ThaiTextAugmenter.model", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/thai2fit.py", - "line": 18 + "parent_class": "ThaiTextAugmenter", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/phayathaibert.py", + "line": 35 }, { - "name": "pythainlp.augment.wordnet.WordNetAug.__init__", + "name": "pythainlp.augment.lm.phayathaibert.ThaiTextAugmenter.processor", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/wordnet.py", - "line": 121 + "parent_class": "ThaiTextAugmenter", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/phayathaibert.py", + "line": 40 }, { - "name": "pythainlp.chat.core.ChatBotModel.__init__", + "name": "pythainlp.augment.lm.wangchanberta.Thai2transformersAug.model_name", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/chat/core.py", - "line": 10 + "parent_class": "Thai2transformersAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/wangchanberta.py", + "line": 28 }, { - "name": "pythainlp.el._multiel.MultiEL.__init__", + "name": "pythainlp.augment.lm.wangchanberta.Thai2transformersAug.target_tokenizer", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/el/_multiel.py", - "line": 7 + "parent_class": "Thai2transformersAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/wangchanberta.py", + "line": 29 }, { - "name": "pythainlp.generate.wangchanglm.WangChanGLM.__init__", + "name": "pythainlp.augment.lm.wangchanberta.Thai2transformersAug.tokenizer", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/wangchanglm.py", - "line": 12 + "parent_class": "Thai2transformersAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/wangchanberta.py", + "line": 30 }, { - "name": "pythainlp.khavee.core.KhaveeVerifier.__init__", + "name": "pythainlp.augment.lm.wangchanberta.Thai2transformersAug.fill_mask", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/khavee/core.py", - "line": 15 + "parent_class": "Thai2transformersAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/wangchanberta.py", + "line": 38 }, { - "name": "pythainlp.phayathaibert.core.ThaiTextProcessor.__init__", + "name": "pythainlp.augment.lm.wangchanberta.Thai2transformersAug.MASK_TOKEN", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/phayathaibert/core.py", - "line": 25 + "parent_class": "Thai2transformersAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/wangchanberta.py", + "line": 44 }, { - "name": "pythainlp.soundex.complete_soundex.CompleteSoundex.__init__", + "name": "pythainlp.augment.lm.wangchanberta.Thai2transformersAug.input_text", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/soundex/complete_soundex.py", - "line": 46 + "parent_class": "Thai2transformersAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/wangchanberta.py", + "line": 50 }, { - "name": "pythainlp.tokenize.attacut.AttacutTokenizer.__init__", + "name": "pythainlp.augment.word2vec.bpemb_wv.BPEmbAug.bpemb_temp", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/attacut.py", - "line": 19 + "parent_class": "BPEmbAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/bpemb_wv.py", + "line": 35 }, { - "name": "pythainlp.tokenize.multi_cut.LatticeString.__init__", + "name": "pythainlp.augment.word2vec.bpemb_wv.BPEmbAug.model", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/multi_cut.py", - "line": 31 + "parent_class": "BPEmbAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/bpemb_wv.py", + "line": 36 }, { - "name": "pythainlp.transliterate.thai2rom.ThaiTransliterator.__init__", + "name": "pythainlp.augment.word2vec.bpemb_wv.BPEmbAug.aug", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", - "line": 22 + "parent_class": "BPEmbAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/bpemb_wv.py", + "line": 47 }, { - "name": "pythainlp.transliterate.thai2rom.Encoder.__init__", + "name": "pythainlp.augment.word2vec.bpemb_wv.BPEmbAug.sentence", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", - "line": 101 + "parent_class": "BPEmbAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/bpemb_wv.py", + "line": 71 }, { - "name": "pythainlp.transliterate.thai2rom.Attn.__init__", + "name": "pythainlp.augment.word2vec.bpemb_wv.BPEmbAug.temp", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", - "line": 165 + "parent_class": "BPEmbAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/bpemb_wv.py", + "line": 72 }, { - "name": "pythainlp.transliterate.thai2rom.AttentionDecoder.__init__", + "name": "pythainlp.augment.word2vec.bpemb_wv.BPEmbAug.temp_new", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", - "line": 211 + "parent_class": "BPEmbAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/bpemb_wv.py", + "line": 73 }, { - "name": "pythainlp.transliterate.thai2rom.Seq2Seq.__init__", + "name": "pythainlp.augment.word2vec.bpemb_wv.BPEmbAug.t", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", - "line": 261 + "parent_class": "BPEmbAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/bpemb_wv.py", + "line": 75 }, { - "name": "pythainlp.transliterate.thai2rom_onnx.ThaiTransliterator_ONNX.__init__", + "name": "pythainlp.augment.word2vec.core.Word2VecAug.model", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", - "line": 21 + "parent_class": "Word2VecAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/core.py", + "line": 34 }, { - "name": "pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.__init__", + "name": "pythainlp.augment.word2vec.core.Word2VecAug.model", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", - "line": 91 + "parent_class": "Word2VecAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/core.py", + "line": 38 }, { - "name": "pythainlp.transliterate.thaig2p.ThaiG2P.__init__", + "name": "pythainlp.augment.word2vec.ltw2v.LTW2VAug.ltw2v_wv", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", - "line": 36 + "parent_class": "LTW2VAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/ltw2v.py", + "line": 30 }, { - "name": "pythainlp.transliterate.thaig2p.Encoder.__init__", + "name": "pythainlp.augment.word2vec.ltw2v.LTW2VAug.aug", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", - "line": 112 + "parent_class": "LTW2VAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/ltw2v.py", + "line": 46 }, { - "name": "pythainlp.transliterate.thaig2p.Attn.__init__", + "name": "pythainlp.augment.word2vec.thai2fit.Thai2fitAug.thai2fit_wv", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", - "line": 178 + "parent_class": "Thai2fitAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/thai2fit.py", + "line": 30 }, { - "name": "pythainlp.transliterate.thaig2p.AttentionDecoder.__init__", + "name": "pythainlp.augment.word2vec.thai2fit.Thai2fitAug.aug", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", - "line": 224 + "parent_class": "Thai2fitAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/thai2fit.py", + "line": 47 }, { - "name": "pythainlp.transliterate.thaig2p.Seq2Seq.__init__", + "name": "pythainlp.augment.wordnet.WordNetAug.list_synsets", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", - "line": 274 + "parent_class": "WordNetAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/wordnet.py", + "line": 140 }, { - "name": "pythainlp.transliterate.w2p.Thai_W2P.__init__", + "name": "pythainlp.augment.wordnet.WordNetAug.p2w_pos", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", - "line": 52 + "parent_class": "WordNetAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/wordnet.py", + "line": 142 }, { - "name": "pythainlp.util.trie.Node.__init__", + "name": "pythainlp.augment.wordnet.WordNetAug.list_synsets", "scope": "public", - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/trie.py", - "line": 52 + "parent_class": "WordNetAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/wordnet.py", + "line": 144 }, { - "name": "pythainlp.__main__.main", + "name": "pythainlp.augment.wordnet.WordNetAug.list_synsets", "scope": "public", - "references": 58, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/__main__.py", - "line": 10 + "parent_class": "WordNetAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/wordnet.py", + "line": 146 }, { - "name": "pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.run", + "name": "pythainlp.augment.wordnet.WordNetAug.temp", "scope": "public", - "references": 53, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", - "line": 115 + "parent_class": "WordNetAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/wordnet.py", + "line": 205 }, { - "name": "pythainlp.el._multiel.MultiEL.load_model", + "name": "pythainlp.chat.core.ChatBotModel.history", "scope": "public", - "references": 18, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/el/_multiel.py", - "line": 12 + "parent_class": "ChatBotModel", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/chat/core.py", + "line": 16 }, { - "name": "pythainlp.transliterate.thai2rom.Encoder.forward", + "name": "pythainlp.chat.core.ChatBotModel.model", "scope": "public", - "references": 15, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", - "line": 119 + "parent_class": "ChatBotModel", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/chat/core.py", + "line": 41 }, { - "name": "pythainlp.transliterate.thai2rom.Attn.forward", + "name": "pythainlp.classify.param_free.GzipModel.training_data", "scope": "public", - "references": 15, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", - "line": 178 + "parent_class": "GzipModel", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/classify/param_free.py", + "line": 36 }, { - "name": "pythainlp.transliterate.thai2rom.AttentionDecoder.forward", + "name": "pythainlp.classify.param_free.GzipModel.cx2_list", "scope": "public", - "references": 15, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", - "line": 233 + "parent_class": "GzipModel", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/classify/param_free.py", + "line": 37 }, { - "name": "pythainlp.transliterate.thai2rom.Seq2Seq.forward", + "name": "pythainlp.classify.param_free.GzipModel.cx2_list", "scope": "public", - "references": 15, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", - "line": 288 + "parent_class": "GzipModel", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/classify/param_free.py", + "line": 109 }, { - "name": "pythainlp.transliterate.thaig2p.Encoder.forward", + "name": "pythainlp.classify.param_free.GzipModel.training_data", "scope": "public", - "references": 15, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", - "line": 130 + "parent_class": "GzipModel", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/classify/param_free.py", + "line": 110 }, { - "name": "pythainlp.transliterate.thaig2p.Attn.forward", + "name": "pythainlp.cli.tag.SubAppBase.args", "scope": "public", - "references": 15, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", - "line": 191 + "parent_class": "SubAppBase", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/tag.py", + "line": 41 }, { - "name": "pythainlp.transliterate.thaig2p.AttentionDecoder.forward", + "name": "pythainlp.cli.tag.POSTaggingApp.separator", "scope": "public", - "references": 15, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", - "line": 246 + "parent_class": "POSTaggingApp", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/tag.py", + "line": 55 }, { - "name": "pythainlp.transliterate.thaig2p.Seq2Seq.forward", + "name": "pythainlp.cli.tag.POSTaggingApp.run", "scope": "public", - "references": 15, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", - "line": 301 + "parent_class": "POSTaggingApp", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/tag.py", + "line": 56 }, { - "name": "pythainlp.tokenize.thai2fit.thai2fit_tokenizer", + "name": "pythainlp.cli.tokenize.SubAppBase.args", "scope": "public", - "references": 12, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/thai2fit.py", - "line": 12 + "parent_class": "SubAppBase", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/tokenize.py", + "line": 77 }, { - "name": "pythainlp.ulmfit.core.merge_wgts", + "name": "pythainlp.corpus.core._ResponseWrapper.status_code", "scope": "public", - "references": 9, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/core.py", - "line": 230 + "parent_class": "_ResponseWrapper", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/core.py", + "line": 41 }, { - "name": "pythainlp.augment.word2vec.bpemb_wv.BPEmbAug.load_w2v", + "name": "pythainlp.corpus.core._ResponseWrapper.headers", "scope": "public", - "references": 6, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/bpemb_wv.py", - "line": 29 + "parent_class": "_ResponseWrapper", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/core.py", + "line": 42 + }, + { + "name": "pythainlp.corpus.core._ResponseWrapper._content", + "scope": "private", + "parent_class": "_ResponseWrapper", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/core.py", + "line": 43 }, { - "name": "pythainlp.augment.word2vec.ltw2v.LTW2VAug.load_w2v", + "name": "pythainlp.el._multiel.MultiEL.model_name", "scope": "public", - "references": 6, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/ltw2v.py", - "line": 28 + "parent_class": "MultiEL", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/el/_multiel.py", + "line": 18 }, { - "name": "pythainlp.augment.word2vec.thai2fit.Thai2fitAug.load_w2v", + "name": "pythainlp.el._multiel.MultiEL.device", "scope": "public", - "references": 6, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/thai2fit.py", + "parent_class": "MultiEL", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/el/_multiel.py", + "line": 19 + }, + { + "name": "pythainlp.el._multiel.MultiEL._bela_run", + "scope": "private", + "parent_class": "MultiEL", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/el/_multiel.py", "line": 29 }, { - "name": "pythainlp.transliterate.thai2rom.Seq2Seq.create_mask", + "name": "pythainlp.generate.core.Unigram.counts", "scope": "public", - "references": 6, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", - "line": 284 + "parent_class": "Unigram", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/core.py", + "line": 43 }, { - "name": "pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.create_mask", + "name": "pythainlp.generate.core.Unigram.counts", "scope": "public", - "references": 6, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", - "line": 111 + "parent_class": "Unigram", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/core.py", + "line": 45 + }, + { + "name": "pythainlp.generate.core.Unigram._word_prob", + "scope": "private", + "parent_class": "Unigram", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/core.py", + "line": 83 }, { - "name": "pythainlp.transliterate.thaig2p.Seq2Seq.create_mask", + "name": "pythainlp.spell.words_spelling_correction.FastTextEncoder.model_dir", "scope": "public", - "references": 6, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", - "line": 297 + "parent_class": "FastTextEncoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/words_spelling_correction.py", + "line": 57 }, { - "name": "pythainlp.tokenize.multi_cut.serialize", + "name": "pythainlp.spell.words_spelling_correction.FastTextEncoder.nn_model_path", "scope": "public", - "references": 4, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/multi_cut.py", - "line": 62 + "parent_class": "FastTextEncoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/words_spelling_correction.py", + "line": 58 }, { - "name": "pythainlp.transliterate.thai2rom.Encoder.init_hidden", + "name": "pythainlp.spell.words_spelling_correction.FastTextEncoder.bucket", "scope": "public", - "references": 4, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", - "line": 153 + "parent_class": "FastTextEncoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/words_spelling_correction.py", + "line": 59 }, { - "name": "pythainlp.transliterate.thaig2p.Encoder.init_hidden", + "name": "pythainlp.spell.words_spelling_correction.FastTextEncoder.nb_words", "scope": "public", - "references": 4, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", - "line": 166 + "parent_class": "FastTextEncoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/words_spelling_correction.py", + "line": 60 }, { - "name": "pythainlp.ulmfit.tokenizer.ThaiTokenizer.add_special_cases", + "name": "pythainlp.spell.words_spelling_correction.FastTextEncoder.minn", "scope": "public", - "references": 4, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/tokenizer.py", - "line": 67 + "parent_class": "FastTextEncoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/words_spelling_correction.py", + "line": 61 }, { - "name": "pythainlp.phayathaibert.core.ThaiTextProcessor._replace_rep", - "scope": "private", - "references": 8, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/phayathaibert/core.py", - "line": 129 + "name": "pythainlp.spell.words_spelling_correction.FastTextEncoder.maxn", + "scope": "public", + "parent_class": "FastTextEncoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/words_spelling_correction.py", + "line": 62 }, { - "name": "pythainlp.ulmfit.preprocess._replace_rep", - "scope": "private", - "references": 8, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/preprocess.py", - "line": 104 + "name": "pythainlp.spell.words_spelling_correction.Words_Spelling_Correction.list_word", + "scope": "public", + "parent_class": "Words_Spelling_Correction", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/words_spelling_correction.py", + "line": 247 }, { - "name": "pythainlp.ulmfit.preprocess._replace_rep", - "scope": "private", - "references": 8, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/preprocess.py", - "line": 227 + "name": "pythainlp.summarize.keybert.KeyBERT.ft_pipeline", + "scope": "public", + "parent_class": "KeyBERT", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/summarize/keybert.py", + "line": 34 }, { - "name": "pythainlp.util.normalize._last_char", - "scope": "private", - "references": 7, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/normalize.py", - "line": 63 + "name": "pythainlp.tag._tag_perceptron.PerceptronTagger.tagdict", + "scope": "public", + "parent_class": "PerceptronTagger", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/_tag_perceptron.py", + "line": 221 }, { - "name": "pythainlp.el._multiel.MultiEL.process_batch", + "name": "pythainlp.tag._tag_perceptron.PerceptronTagger.classes", "scope": "public", - "references": 3, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/el/_multiel.py", - "line": 21 + "parent_class": "PerceptronTagger", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/_tag_perceptron.py", + "line": 222 }, { - "name": "pythainlp.transliterate.w2p.Thai_W2P._grucell", + "name": "pythainlp.tag.crfchunk.CRFchunk._model_file_ctx", "scope": "private", - "references": 3, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", - "line": 100 + "parent_class": "CRFchunk", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/crfchunk.py", + "line": 90 }, { - "name": "pythainlp.wangchanberta.core._get_tokenizer", + "name": "pythainlp.tag.crfchunk.CRFchunk._model_file_ctx", "scope": "private", - "references": 3, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py", - "line": 16 + "parent_class": "CRFchunk", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/crfchunk.py", + "line": 112 }, { - "name": "pythainlp.wangchanberta.core.ThaiNameTagger._clear_tag", - "scope": "private", - "references": 3, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py", - "line": 65 + "name": "pythainlp.tag.named_entity.NER.engine", + "scope": "public", + "parent_class": "NER", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/named_entity.py", + "line": 51 }, { - "name": "pythainlp.coref._fastcoref.FastCoref._to_json", - "scope": "private", - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/coref/_fastcoref.py", - "line": 29 + "name": "pythainlp.tag.named_entity.NER.engine", + "scope": "public", + "parent_class": "NER", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/named_entity.py", + "line": 55 }, { - "name": "pythainlp.tokenize.budoux._init_parser", - "scope": "private", - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/budoux.py", - "line": 22 + "name": "pythainlp.tag.named_entity.NER.engine", + "scope": "public", + "parent_class": "NER", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/named_entity.py", + "line": 61 }, { - "name": "pythainlp.tokenize.etcc._cut_etcc", - "scope": "private", - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/etcc.py", - "line": 32 + "name": "pythainlp.tag.named_entity.NER.engine", + "scope": "public", + "parent_class": "NER", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/named_entity.py", + "line": 65 }, { - "name": "pythainlp.tokenize.multi_cut.LatticeString.__new__", + "name": "pythainlp.tag.named_entity.NER.engine", "scope": "public", - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/multi_cut.py", - "line": 28 + "parent_class": "NER", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/named_entity.py", + "line": 73 }, { - "name": "pythainlp.tokenize.nlpo3._ensure_default_dict_loaded", + "name": "pythainlp.tag.named_entity.NER.engine", + "scope": "public", + "parent_class": "NER", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/named_entity.py", + "line": 78 + }, + { + "name": "pythainlp.tag.thainer.ThaiNameTagger.pos_tag_name", + "scope": "public", + "parent_class": "ThaiNameTagger", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/thainer.py", + "line": 128 + }, + { + "name": "pythainlp.tag.wangchanberta_onnx.WngchanBerta_ONNX._json", "scope": "private", - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/nlpo3.py", + "parent_class": "WngchanBerta_ONNX", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/wangchanberta_onnx.py", + "line": 55 + }, + { + "name": "pythainlp.tag.wangchanberta_onnx.WngchanBerta_ONNX.id2tag", + "scope": "public", + "parent_class": "WngchanBerta_ONNX", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/wangchanberta_onnx.py", + "line": 56 + }, + { + "name": "pythainlp.tag.wangchanberta_onnx.WngchanBerta_ONNX._s", + "scope": "private", + "parent_class": "WngchanBerta_ONNX", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/wangchanberta_onnx.py", + "line": 101 + }, + { + "name": "pythainlp.tokenize.attacut.AttacutTokenizer._MODEL_NAME", + "scope": "private", + "parent_class": "AttacutTokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/attacut.py", "line": 23 }, { - "name": "pythainlp.transliterate.w2p._load_vocab", + "name": "pythainlp.tokenize.attacut.AttacutTokenizer._MODEL_NAME", "scope": "private", - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", - "line": 41 + "parent_class": "AttacutTokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/attacut.py", + "line": 26 }, { - "name": "pythainlp.transliterate.w2p.Thai_W2P._sigmoid", + "name": "pythainlp.tokenize.attacut.AttacutTokenizer._tokenizer", "scope": "private", - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", - "line": 97 + "parent_class": "AttacutTokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/attacut.py", + "line": 28 }, { - "name": "pythainlp.wangchanberta.core.ThaiNameTagger._IOB", + "name": "pythainlp.tokenize.core.Tokenizer.__trie_dict", "scope": "private", - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py", - "line": 60 + "parent_class": "Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/core.py", + "line": 944 }, { - "name": "pythainlp.wangchanberta.core.NamedEntityRecognition._fix_span_error", + "name": "pythainlp.tokenize.core.Tokenizer.__trie_dict", "scope": "private", - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py", - "line": 161 + "parent_class": "Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/core.py", + "line": 946 }, { - "name": "pythainlp.chat.core.ChatBotModel.reset_chat", + "name": "pythainlp.tokenize.core.Tokenizer.__engine", + "scope": "private", + "parent_class": "Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/core.py", + "line": 996 + }, + { + "name": "pythainlp.tokenize.han_solo.Featurizer.N", "scope": "public", - "references": 1, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/chat/core.py", - "line": 14 + "parent_class": "Featurizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/han_solo.py", + "line": 61 }, { - "name": "pythainlp.khavee.core.KhaveeVerifier.check_karu_lahu", + "name": "pythainlp.tokenize.han_solo.Featurizer.delimiter", "scope": "public", - "references": 1, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/khavee/core.py", - "line": 359 + "parent_class": "Featurizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/han_solo.py", + "line": 62 }, { - "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer._switch_to_input_mode", - "scope": "private", - "references": 1, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", - "line": 397 + "name": "pythainlp.tokenize.han_solo.Featurizer.radius", + "scope": "public", + "parent_class": "Featurizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/han_solo.py", + "line": 63 }, { - "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer._switch_to_target_mode", + "name": "pythainlp.tokenize.longest.LongestMatchTokenizer.__trie", "scope": "private", - "references": 1, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", - "line": 400 - } - ], - "functions_incomplete_hints": [ + "parent_class": "LongestMatchTokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/longest.py", + "line": 52 + }, { - "name": "pythainlp.tokenize.nercut.segment", + "name": "pythainlp.tokenize.multi_cut.LatticeString.unique", "scope": "public", - "params": "2/3", - "return": true, - "references": 350, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/nercut.py", - "line": 22 + "parent_class": "LatticeString", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/multi_cut.py", + "line": 45 }, { - "name": "pythainlp.classify.param_free.GzipModel.load", + "name": "pythainlp.tokenize.multi_cut.LatticeString.multi", "scope": "public", - "params": "1/1", - "return": false, - "references": 334, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/classify/param_free.py", - "line": 101 + "parent_class": "LatticeString", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/multi_cut.py", + "line": 47 }, { - "name": "pythainlp.transliterate.thaig2p_v2.transliterate", + "name": "pythainlp.translate.core.Translate.model", "scope": "public", - "params": "1/2", - "return": true, - "references": 127, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p_v2.py", - "line": 41 + "parent_class": "Translate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/core.py", + "line": 75 }, { - "name": "pythainlp.transliterate.umt5_thaig2p.transliterate", + "name": "pythainlp.translate.core.Translate.model", "scope": "public", - "params": "1/2", - "return": true, - "references": 127, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/umt5_thaig2p.py", - "line": 41 + "parent_class": "Translate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/core.py", + "line": 79 }, { - "name": "pythainlp.transliterate.wunsen.WunsenTransliterate.transliterate", + "name": "pythainlp.translate.core.Translate.model", "scope": "public", - "params": "5/5", - "return": false, - "references": 127, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py", + "parent_class": "Translate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/core.py", + "line": 83 + }, + { + "name": "pythainlp.translate.core.Translate.model", + "scope": "public", + "parent_class": "Translate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/core.py", + "line": 87 + }, + { + "name": "pythainlp.translate.core.Translate.model", + "scope": "public", + "parent_class": "Translate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/core.py", + "line": 91 + }, + { + "name": "pythainlp.translate.core.Translate.model", + "scope": "public", + "parent_class": "Translate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/core.py", + "line": 95 + }, + { + "name": "pythainlp.translate.en_th.EnThTranslator._model", + "scope": "private", + "parent_class": "EnThTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/en_th.py", + "line": 89 + }, + { + "name": "pythainlp.translate.small100.Small100Translator.pretrained", + "scope": "public", + "parent_class": "Small100Translator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/small100.py", + "line": 36 + }, + { + "name": "pythainlp.translate.small100.Small100Translator.model", + "scope": "public", + "parent_class": "Small100Translator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/small100.py", "line": 37 }, { - "name": "pythainlp.augment.lm.fasttext.FastTextAug.__init__", + "name": "pythainlp.translate.small100.Small100Translator.tgt_lang", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/fasttext.py", - "line": 17 + "parent_class": "Small100Translator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/small100.py", + "line": 40 }, { - "name": "pythainlp.augment.word2vec.bpemb_wv.BPEmbAug.__init__", + "name": "pythainlp.translate.small100.Small100Translator.model", "scope": "public", - "params": "3/3", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/bpemb_wv.py", - "line": 16 + "parent_class": "Small100Translator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/small100.py", + "line": 42 }, { - "name": "pythainlp.classify.param_free.GzipModel.__init__", + "name": "pythainlp.translate.small100.Small100Translator.tokenizer", "scope": "public", - "params": "2/2", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/classify/param_free.py", - "line": 24 + "parent_class": "Small100Translator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/small100.py", + "line": 74 }, { - "name": "pythainlp.coref._fastcoref.FastCoref.__init__", + "name": "pythainlp.translate.small100.Small100Translator.tgt_lang", "scope": "public", - "params": "2/4", - "return": true, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/coref/_fastcoref.py", - "line": 8 + "parent_class": "Small100Translator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/small100.py", + "line": 77 }, { - "name": "pythainlp.coref.han_coref.HanCoref.__init__", + "name": "pythainlp.translate.small100.Small100Translator.translated", "scope": "public", - "params": "1/2", - "return": true, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/coref/han_coref.py", - "line": 10 + "parent_class": "Small100Translator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/small100.py", + "line": 78 }, { - "name": "pythainlp.el.core.EntityLinker.__init__", + "name": "pythainlp.translate.th_fr.ThFrTranslator.tokenizer_thfr", "scope": "public", - "params": "3/3", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/el/core.py", - "line": 10 + "parent_class": "ThFrTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/th_fr.py", + "line": 49 }, { - "name": "pythainlp.generate.core.Unigram.__init__", + "name": "pythainlp.translate.th_fr.ThFrTranslator.model_thfr", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/core.py", - "line": 33 + "parent_class": "ThFrTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/th_fr.py", + "line": 50 }, { - "name": "pythainlp.generate.core.Bigram.__init__", + "name": "pythainlp.translate.th_fr.ThFrTranslator.model_thfr", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/core.py", - "line": 119 + "parent_class": "ThFrTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/th_fr.py", + "line": 52 }, { - "name": "pythainlp.generate.core.Trigram.__init__", + "name": "pythainlp.translate.th_fr.ThFrTranslator.translated", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/core.py", - "line": 206 + "parent_class": "ThFrTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/th_fr.py", + "line": 73 }, { - "name": "pythainlp.spell.pn.NorvigSpellChecker.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.sp_model_kwargs", "scope": "public", - "params": "5/5", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/pn.py", - "line": 123 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 156 }, { - "name": "pythainlp.summarize.freq.FrequencySummarizer.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.language_codes", "scope": "public", - "params": "2/2", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/summarize/freq.py", - "line": 20 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 160 }, { - "name": "pythainlp.summarize.keybert.KeyBERT.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.lang_code_to_token", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/summarize/keybert.py", - "line": 27 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 162 }, { - "name": "pythainlp.summarize.mt5.mT5Summarizer.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.vocab_file", "scope": "public", - "params": "7/7", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/summarize/mt5.py", - "line": 12 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 190 }, { - "name": "pythainlp.tag.crfchunk.CRFchunk.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.encoder", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/crfchunk.py", - "line": 77 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 194 }, { - "name": "pythainlp.tokenize.core.Tokenizer.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.decoder", "scope": "public", - "params": "4/4", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/core.py", - "line": 925 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 195 }, { - "name": "pythainlp.tokenize.longest.LongestMatchTokenizer.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.spm_file", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/longest.py", - "line": 49 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 196 }, { - "name": "pythainlp.translate.en_th.EnThTranslator.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.sp_model", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/en_th.py", - "line": 68 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 197 }, { - "name": "pythainlp.translate.en_th.ThEnTranslator.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.encoder_size", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/en_th.py", - "line": 124 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 199 }, { - "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.lang_token_to_id", "scope": "public", - "params": "1/11", - "return": true, - "references": 113, - "test_suite": "unknown", - "priority": "medium", + "parent_class": "SMALL100Tokenizer", "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", - "line": 125 + "line": 201 }, { - "name": "pythainlp.transliterate.thaig2p_v2.ThaiG2P.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.lang_code_to_id", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p_v2.py", - "line": 27 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 205 }, { - "name": "pythainlp.transliterate.umt5_thaig2p.Umt5ThaiG2P.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.id_to_lang_token", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/umt5_thaig2p.py", - "line": 27 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 209 + }, + { + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer._tgt_lang", + "scope": "private", + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 213 }, { - "name": "pythainlp.ulmfit.tokenizer.BaseTokenizer.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.cur_lang_id", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/tokenizer.py", - "line": 16 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 214 }, { - "name": "pythainlp.ulmfit.tokenizer.ThaiTokenizer.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.num_madeup_words", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/tokenizer.py", - "line": 32 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 217 + }, + { + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer._tgt_lang", + "scope": "private", + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 230 }, { - "name": "pythainlp.util.trie.Trie.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.__dict__", "scope": "public", - "params": "1/1", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/trie.py", - "line": 56 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 353 }, { - "name": "pythainlp.wangchanberta.core.ThaiNameTagger.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.sp_model_kwargs", "scope": "public", - "params": "2/2", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py", - "line": 35 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 357 }, { - "name": "pythainlp.wsd.core._SentenceTransformersModel.__init__", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.sp_model", "scope": "public", - "params": "2/2", - "return": false, - "references": 113, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py", - "line": 26 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 359 }, { - "name": "pythainlp.augment.lm.wangchanberta.Thai2transformersAug.generate", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.tgt_lang", "scope": "public", - "params": "2/2", - "return": false, - "references": 37, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/wangchanberta.py", - "line": 34 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 396 }, { - "name": "pythainlp.classify.param_free.GzipModel.save", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.tgt_lang", "scope": "public", - "params": "1/1", - "return": false, - "references": 37, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/classify/param_free.py", - "line": 89 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 409 }, { - "name": "pythainlp.util.date.convert_years", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.prefix_tokens", "scope": "public", - "params": "1/3", - "return": true, - "references": 24, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py", - "line": 119 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 417 }, { - "name": "pythainlp.summarize.keybert.KeyBERT.extract_keywords", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.suffix_tokens", "scope": "public", - "params": "6/7", - "return": true, - "references": 20, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/summarize/keybert.py", - "line": 39 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 418 }, { - "name": "pythainlp.spell.symspellpy.correct_sent", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.cur_lang_id", "scope": "public", - "params": "1/2", - "return": true, - "references": 19, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/symspellpy.py", - "line": 111 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 424 }, { - "name": "pythainlp.chat.core.ChatBotModel.load_model", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.prefix_tokens", "scope": "public", - "params": "6/7", - "return": false, - "references": 18, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/chat/core.py", - "line": 18 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 425 }, { - "name": "pythainlp.generate.wangchanglm.WangChanGLM.load_model", + "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer.suffix_tokens", "scope": "public", - "params": "6/7", - "return": false, - "references": 18, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/wangchanglm.py", - "line": 26 + "parent_class": "SMALL100Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 426 }, { - "name": "pythainlp.util.emojiconv.emoji_to_thai", + "name": "pythainlp.translate.zh_th.ThZhTranslator.tokenizer_thzh", "scope": "public", - "params": "1/2", - "return": true, - "references": 15, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/emojiconv.py", - "line": 1835 + "parent_class": "ThZhTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/zh_th.py", + "line": 43 }, { - "name": "pythainlp.util.date.thai_strptime", + "name": "pythainlp.translate.zh_th.ThZhTranslator.model_thzh", "scope": "public", - "params": "4/5", - "return": false, - "references": 12, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py", - "line": 212 + "parent_class": "ThZhTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/zh_th.py", + "line": 44 }, { - "name": "pythainlp.ulmfit.core.document_vector", + "name": "pythainlp.translate.zh_th.ThZhTranslator.model_thzh", "scope": "public", - "params": "2/4", - "return": false, - "references": 11, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/core.py", - "line": 175 + "parent_class": "ThZhTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/zh_th.py", + "line": 46 }, { - "name": "pythainlp.augment.wordnet.postype2wordnet", + "name": "pythainlp.translate.zh_th.ThZhTranslator.translated", "scope": "public", - "params": "2/2", - "return": false, - "references": 6, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/wordnet.py", - "line": 104 + "parent_class": "ThZhTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/zh_th.py", + "line": 67 }, { - "name": "pythainlp.ulmfit.tokenizer.BaseTokenizer.add_special_cases", + "name": "pythainlp.translate.zh_th.ZhThTranslator.tokenizer_zhth", "scope": "public", - "params": "1/1", - "return": false, - "references": 4, - "test_suite": "unknown", - "priority": "medium", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/tokenizer.py", - "line": 22 + "parent_class": "ZhThTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/zh_th.py", + "line": 99 }, { - "name": "pythainlp.transliterate.thai2rom.ThaiTransliterator._prepare_sequence_in", - "scope": "private", - "params": "1/1", - "return": false, - "references": 12, - "test_suite": "unknown", - "priority": "low", + "name": "pythainlp.translate.zh_th.ZhThTranslator.model_zhth", + "scope": "public", + "parent_class": "ZhThTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/zh_th.py", + "line": 100 + }, + { + "name": "pythainlp.translate.zh_th.ZhThTranslator.translated", + "scope": "public", + "parent_class": "ZhThTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/zh_th.py", + "line": 123 + }, + { + "name": "pythainlp.transliterate.thai2rom.Encoder.hidden_size", + "scope": "public", + "parent_class": "Encoder", "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", - "line": 61 + "line": 124 }, { - "name": "pythainlp.transliterate.thai2rom_onnx.ThaiTransliterator_ONNX._prepare_sequence_in", - "scope": "private", - "params": "1/1", - "return": false, - "references": 12, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", - "line": 59 + "name": "pythainlp.transliterate.thai2rom.Encoder.character_embedding", + "scope": "public", + "parent_class": "Encoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 125 }, { - "name": "pythainlp.transliterate.thaig2p.ThaiG2P._prepare_sequence_in", - "scope": "private", - "params": "1/1", - "return": false, - "references": 12, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", - "line": 71 + "name": "pythainlp.transliterate.thai2rom.Encoder.rnn", + "scope": "public", + "parent_class": "Encoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 128 }, { - "name": "pythainlp.benchmarks.word_tokenization._find_word_boundaries", - "scope": "private", - "params": "0/1", - "return": true, - "references": 5, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/benchmarks/word_tokenization.py", - "line": 239 + "name": "pythainlp.transliterate.thai2rom.Encoder.dropout", + "scope": "public", + "parent_class": "Encoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 135 }, { - "name": "pythainlp.transliterate.w2p.Thai_W2P._gru", - "scope": "private", - "params": "0/7", - "return": true, - "references": 5, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", - "line": 121 + "name": "pythainlp.transliterate.thai2rom.Attn.method", + "scope": "public", + "parent_class": "Attn", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 188 }, { - "name": "pythainlp.tag.thainer._doc2features", - "scope": "private", - "params": "0/2", - "return": true, - "references": 4, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/thainer.py", - "line": 25 + "name": "pythainlp.transliterate.thai2rom.Attn.hidden_size", + "scope": "public", + "parent_class": "Attn", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 189 }, { - "name": "pythainlp.generate.wangchanglm.WangChanGLM.gen_instruct", + "name": "pythainlp.transliterate.thai2rom.Attn.attn", "scope": "public", - "params": "9/9", - "return": false, - "references": 3, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/wangchanglm.py", - "line": 68 + "parent_class": "Attn", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 194 }, { - "name": "pythainlp.corpus.wordnet.custom_lemmas", + "name": "pythainlp.transliterate.thai2rom.Attn.attn", "scope": "public", - "params": "1/2", - "return": true, - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/wordnet.py", - "line": 437 + "parent_class": "Attn", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 197 }, { - "name": "pythainlp.generate.wangchanglm.WangChanGLM.instruct_generate", + "name": "pythainlp.transliterate.thai2rom.Attn.other", "scope": "public", - "params": "9/10", - "return": false, - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/wangchanglm.py", - "line": 124 + "parent_class": "Attn", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 198 }, { - "name": "pythainlp.summarize.freq.FrequencySummarizer.__rank", - "scope": "private", - "params": "1/2", - "return": false, - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/summarize/freq.py", - "line": 26 + "name": "pythainlp.transliterate.thai2rom.AttentionDecoder.vocabulary_size", + "scope": "public", + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 247 }, { - "name": "pythainlp.translate.tokenization_small100.save_json", + "name": "pythainlp.transliterate.thai2rom.AttentionDecoder.hidden_size", "scope": "public", - "params": "1/2", - "return": true, - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", - "line": 433 + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 248 }, { - "name": "pythainlp.wsd.core._SentenceTransformersModel.change_device", + "name": "pythainlp.transliterate.thai2rom.AttentionDecoder.character_embedding", "scope": "public", - "params": "1/1", - "return": false, - "references": 2, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py", - "line": 37 + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 249 }, { - "name": "pythainlp.translate.tokenization_small100.SMALL100Tokenizer._build_translation_inputs", - "scope": "private", - "params": "1/2", - "return": false, - "references": 1, - "test_suite": "unknown", - "priority": "low", - "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", - "line": 384 - } - ] + "name": "pythainlp.transliterate.thai2rom.AttentionDecoder.rnn", + "scope": "public", + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 252 + }, + { + "name": "pythainlp.transliterate.thai2rom.AttentionDecoder.attn", + "scope": "public", + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 259 + }, + { + "name": "pythainlp.transliterate.thai2rom.AttentionDecoder.linear", + "scope": "public", + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 260 + }, + { + "name": "pythainlp.transliterate.thai2rom.AttentionDecoder.dropout", + "scope": "public", + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 262 + }, + { + "name": "pythainlp.transliterate.thai2rom.Seq2Seq.encoder", + "scope": "public", + "parent_class": "Seq2Seq", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 308 + }, + { + "name": "pythainlp.transliterate.thai2rom.Seq2Seq.decoder", + "scope": "public", + "parent_class": "Seq2Seq", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 309 + }, + { + "name": "pythainlp.transliterate.thai2rom.Seq2Seq.pad_idx", + "scope": "public", + "parent_class": "Seq2Seq", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 310 + }, + { + "name": "pythainlp.transliterate.thai2rom.Seq2Seq.target_start_token", + "scope": "public", + "parent_class": "Seq2Seq", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 311 + }, + { + "name": "pythainlp.transliterate.thai2rom.Seq2Seq.target_end_token", + "scope": "public", + "parent_class": "Seq2Seq", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 312 + }, + { + "name": "pythainlp.transliterate.thai2rom.Seq2Seq.max_length", + "scope": "public", + "parent_class": "Seq2Seq", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 313 + }, + { + "name": "pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.encoder", + "scope": "public", + "parent_class": "Seq2Seq_ONNX", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", + "line": 106 + }, + { + "name": "pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.decoder", + "scope": "public", + "parent_class": "Seq2Seq_ONNX", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", + "line": 107 + }, + { + "name": "pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.pad_idx", + "scope": "public", + "parent_class": "Seq2Seq_ONNX", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", + "line": 108 + }, + { + "name": "pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.target_start_token", + "scope": "public", + "parent_class": "Seq2Seq_ONNX", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", + "line": 109 + }, + { + "name": "pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.target_end_token", + "scope": "public", + "parent_class": "Seq2Seq_ONNX", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", + "line": 110 + }, + { + "name": "pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.max_length", + "scope": "public", + "parent_class": "Seq2Seq_ONNX", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", + "line": 111 + }, + { + "name": "pythainlp.transliterate.thai2rom_onnx.Seq2Seq_ONNX.target_vocab_size", + "scope": "public", + "parent_class": "Seq2Seq_ONNX", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", + "line": 113 + }, + { + "name": "pythainlp.transliterate.thaig2p.Encoder.hidden_size", + "scope": "public", + "parent_class": "Encoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 135 + }, + { + "name": "pythainlp.transliterate.thaig2p.Encoder.character_embedding", + "scope": "public", + "parent_class": "Encoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 136 + }, + { + "name": "pythainlp.transliterate.thaig2p.Encoder.rnn", + "scope": "public", + "parent_class": "Encoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 139 + }, + { + "name": "pythainlp.transliterate.thaig2p.Encoder.dropout", + "scope": "public", + "parent_class": "Encoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 146 + }, + { + "name": "pythainlp.transliterate.thaig2p.Attn.method", + "scope": "public", + "parent_class": "Attn", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 205 + }, + { + "name": "pythainlp.transliterate.thaig2p.Attn.hidden_size", + "scope": "public", + "parent_class": "Attn", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 206 + }, + { + "name": "pythainlp.transliterate.thaig2p.Attn.attn", + "scope": "public", + "parent_class": "Attn", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 212 + }, + { + "name": "pythainlp.transliterate.thaig2p.AttentionDecoder.vocabulary_size", + "scope": "public", + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 264 + }, + { + "name": "pythainlp.transliterate.thaig2p.AttentionDecoder.hidden_size", + "scope": "public", + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 265 + }, + { + "name": "pythainlp.transliterate.thaig2p.AttentionDecoder.character_embedding", + "scope": "public", + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 266 + }, + { + "name": "pythainlp.transliterate.thaig2p.AttentionDecoder.rnn", + "scope": "public", + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 269 + }, + { + "name": "pythainlp.transliterate.thaig2p.AttentionDecoder.attn", + "scope": "public", + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 276 + }, + { + "name": "pythainlp.transliterate.thaig2p.AttentionDecoder.linear", + "scope": "public", + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 277 + }, + { + "name": "pythainlp.transliterate.thaig2p.AttentionDecoder.dropout", + "scope": "public", + "parent_class": "AttentionDecoder", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 279 + }, + { + "name": "pythainlp.transliterate.thaig2p.Seq2Seq.encoder", + "scope": "public", + "parent_class": "Seq2Seq", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 325 + }, + { + "name": "pythainlp.transliterate.thaig2p.Seq2Seq.decoder", + "scope": "public", + "parent_class": "Seq2Seq", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 326 + }, + { + "name": "pythainlp.transliterate.thaig2p.Seq2Seq.pad_idx", + "scope": "public", + "parent_class": "Seq2Seq", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 327 + }, + { + "name": "pythainlp.transliterate.thaig2p.Seq2Seq.target_start_token", + "scope": "public", + "parent_class": "Seq2Seq", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 328 + }, + { + "name": "pythainlp.transliterate.thaig2p.Seq2Seq.target_end_token", + "scope": "public", + "parent_class": "Seq2Seq", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 329 + }, + { + "name": "pythainlp.transliterate.thaig2p.Seq2Seq.max_length", + "scope": "public", + "parent_class": "Seq2Seq", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 330 + }, + { + "name": "pythainlp.transliterate.umt5_thaig2p.Umt5ThaiG2P.pipe", + "scope": "public", + "parent_class": "Umt5ThaiG2P", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/umt5_thaig2p.py", + "line": 35 + }, + { + "name": "pythainlp.transliterate.w2p.Thai_W2P.checkpoint", + "scope": "public", + "parent_class": "Thai_W2P", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", + "line": 89 + }, + { + "name": "pythainlp.transliterate.w2p.Thai_W2P.word", + "scope": "public", + "parent_class": "Thai_W2P", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", + "line": 172 + }, + { + "name": "pythainlp.transliterate.w2p.Thai_W2P.word", + "scope": "public", + "parent_class": "Thai_W2P", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", + "line": 173 + }, + { + "name": "pythainlp.transliterate.wunsen.WunsenTransliterate.jp_input", + "scope": "public", + "parent_class": "WunsenTransliterate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py", + "line": 119 + }, + { + "name": "pythainlp.transliterate.wunsen.WunsenTransliterate.zh_sandhi", + "scope": "public", + "parent_class": "WunsenTransliterate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py", + "line": 120 + }, + { + "name": "pythainlp.transliterate.wunsen.WunsenTransliterate.system", + "scope": "public", + "parent_class": "WunsenTransliterate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py", + "line": 121 + }, + { + "name": "pythainlp.transliterate.wunsen.WunsenTransliterate.jp_input", + "scope": "public", + "parent_class": "WunsenTransliterate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py", + "line": 123 + }, + { + "name": "pythainlp.transliterate.wunsen.WunsenTransliterate.zh_sandhi", + "scope": "public", + "parent_class": "WunsenTransliterate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py", + "line": 124 + }, + { + "name": "pythainlp.transliterate.wunsen.WunsenTransliterate.system", + "scope": "public", + "parent_class": "WunsenTransliterate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py", + "line": 125 + }, + { + "name": "pythainlp.transliterate.wunsen.WunsenTransliterate.jp_input", + "scope": "public", + "parent_class": "WunsenTransliterate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py", + "line": 127 + }, + { + "name": "pythainlp.transliterate.wunsen.WunsenTransliterate.zh_sandhi", + "scope": "public", + "parent_class": "WunsenTransliterate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py", + "line": 128 + }, + { + "name": "pythainlp.transliterate.wunsen.WunsenTransliterate.system", + "scope": "public", + "parent_class": "WunsenTransliterate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py", + "line": 129 + }, + { + "name": "pythainlp.transliterate.wunsen.WunsenTransliterate.lang", + "scope": "public", + "parent_class": "WunsenTransliterate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py", + "line": 134 + }, + { + "name": "pythainlp.transliterate.wunsen.WunsenTransliterate.thap_value", + "scope": "public", + "parent_class": "WunsenTransliterate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/wunsen.py", + "line": 145 + }, + { + "name": "pythainlp.util.trie.Trie.words", + "scope": "public", + "parent_class": "Trie", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/trie.py", + "line": 60 + }, + { + "name": "pythainlp.util.trie.Trie.root", + "scope": "public", + "parent_class": "Trie", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/trie.py", + "line": 61 + }, + { + "name": "pythainlp.wangchanberta.core.ThaiNameTagger.sent_ner", + "scope": "public", + "parent_class": "ThaiNameTagger", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py", + "line": 108 + }, + { + "name": "pythainlp.wangchanberta.core.ThaiNameTagger.sent_ner", + "scope": "public", + "parent_class": "ThaiNameTagger", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py", + "line": 114 + }, + { + "name": "pythainlp.wangchanberta.core.ThaiNameTagger.sent_ner", + "scope": "public", + "parent_class": "ThaiNameTagger", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py", + "line": 122 + }, + { + "name": "pythainlp.word_vector.core.WordVector.tokenize", + "scope": "public", + "parent_class": "WordVector", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/word_vector/core.py", + "line": 68 + }, + { + "name": "pythainlp.word_vector.core.WordVector.tokenize", + "scope": "public", + "parent_class": "WordVector", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/word_vector/core.py", + "line": 70 + }, + { + "name": "pythainlp.wsd.core._SentenceTransformersModel.device", + "scope": "public", + "parent_class": "_SentenceTransformersModel", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py", + "line": 40 + }, + { + "name": "pythainlp.wsd.core._SentenceTransformersModel.model", + "scope": "public", + "parent_class": "_SentenceTransformersModel", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py", + "line": 41 + } + ], + "module_variables_no_hints": [ + { + "name": "pythainlp.ancient.aksonhan._dict_aksonhan", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ancient/aksonhan.py", + "line": 13 + }, + { + "name": "pythainlp.ancient.aksonhan.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ancient/aksonhan.py", + "line": 18 + }, + { + "name": "pythainlp.ancient.aksonhan.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ancient/aksonhan.py", + "line": 19 + }, + { + "name": "pythainlp.ancient.aksonhan.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ancient/aksonhan.py", + "line": 20 + }, + { + "name": "pythainlp.ancient.aksonhan._set_aksonhan", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ancient/aksonhan.py", + "line": 21 + }, + { + "name": "pythainlp.ancient.aksonhan._trie", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ancient/aksonhan.py", + "line": 22 + }, + { + "name": "pythainlp.ancient.aksonhan._tokenizer", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ancient/aksonhan.py", + "line": 23 + }, + { + "name": "pythainlp.ancient.aksonhan._dict_thai", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ancient/aksonhan.py", + "line": 24 + }, + { + "name": "pythainlp.augment.lm.phayathaibert._MODEL_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/phayathaibert.py", + "line": 15 + }, + { + "name": "pythainlp.augment.lm.wangchanberta.model_name", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/wangchanberta.py", + "line": 11 + }, + { + "name": "pythainlp.augment.word2vec.ltw2v.Word2VecAug", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/ltw2v.py", + "line": 14 + }, + { + "name": "pythainlp.augment.word2vec.thai2fit.Word2VecAug", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/thai2fit.py", + "line": 14 + }, + { + "name": "pythainlp.augment.wordnet.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/wordnet.py", + "line": 8 + }, + { + "name": "pythainlp.augment.wordnet.orchid", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/wordnet.py", + "line": 23 + }, + { + "name": "pythainlp.benchmarks.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/benchmarks/__init__.py", + "line": 6 + }, + { + "name": "pythainlp.benchmarks.word_tokenization.SEPARATOR", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/benchmarks/word_tokenization.py", + "line": 13 + }, + { + "name": "pythainlp.benchmarks.word_tokenization.SURROUNDING_SEPS_RX", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/benchmarks/word_tokenization.py", + "line": 16 + }, + { + "name": "pythainlp.benchmarks.word_tokenization.MULTIPLE_SEPS_RX", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/benchmarks/word_tokenization.py", + "line": 21 + }, + { + "name": "pythainlp.benchmarks.word_tokenization.TAG_RX", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/benchmarks/word_tokenization.py", + "line": 24 + }, + { + "name": "pythainlp.benchmarks.word_tokenization.TAILING_SEP_RX", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/benchmarks/word_tokenization.py", + "line": 27 + }, + { + "name": "pythainlp.cli.stdout", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/__init__.py", + "line": 14 + }, + { + "name": "pythainlp.cli.stderr", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/__init__.py", + "line": 15 + }, + { + "name": "pythainlp.cli.COMMANDS", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/__init__.py", + "line": 18 + }, + { + "name": "pythainlp.cli.CLI_NAME", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/__init__.py", + "line": 22 + }, + { + "name": "pythainlp.cli.COMMAND_MAP", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/__init__.py", + "line": 47 + }, + { + "name": "pythainlp.cli.command", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/__init__.py", + "line": 58 + }, + { + "name": "pythainlp.cli.tokenize.DEFAULT_SENT_TOKEN_SEPARATOR", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/tokenize.py", + "line": 25 + }, + { + "name": "pythainlp.cli.tokenize.DEFAULT_SUBWORD_TOKEN_SEPARATOR", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/tokenize.py", + "line": 26 + }, + { + "name": "pythainlp.cli.tokenize.DEFAULT_SYLLABLE_TOKEN_SEPARATOR", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/tokenize.py", + "line": 27 + }, + { + "name": "pythainlp.cli.tokenize.DEFAULT_WORD_TOKEN_SEPARATOR", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/tokenize.py", + "line": 28 + }, + { + "name": "pythainlp.coref.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/coref/__init__.py", + "line": 6 + }, + { + "name": "pythainlp.coref.core._MODEL", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/coref/core.py", + "line": 8 + }, + { + "name": "pythainlp.corpus.common.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/common.py", + "line": 15 + }, + { + "name": "pythainlp.corpus.common._THAI_COUNTRIES_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/common.py", + "line": 37 + }, + { + "name": "pythainlp.corpus.common._THAI_THAILAND_PROVINCES_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/common.py", + "line": 41 + }, + { + "name": "pythainlp.corpus.common._THAI_SYLLABLES_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/common.py", + "line": 44 + }, + { + "name": "pythainlp.corpus.common._THAI_WORDS_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/common.py", + "line": 47 + }, + { + "name": "pythainlp.corpus.common._THAI_STOPWORDS_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/common.py", + "line": 50 + }, + { + "name": "pythainlp.corpus.common._THAI_NEGATIONS_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/common.py", + "line": 53 + }, + { + "name": "pythainlp.corpus.common._THAI_PROFANITY_WORDS_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/common.py", + "line": 56 + }, + { + "name": "pythainlp.corpus.common._THAI_FAMLIY_NAMES_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/common.py", + "line": 59 + }, + { + "name": "pythainlp.corpus.common._THAI_FEMALE_NAMES_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/common.py", + "line": 61 + }, + { + "name": "pythainlp.corpus.common._THAI_MALE_NAMES_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/common.py", + "line": 63 + }, + { + "name": "pythainlp.corpus.core._CHECK_MODE", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/core.py", + "line": 25 + }, + { + "name": "pythainlp.corpus.core._USER_AGENT", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/core.py", + "line": 26 + }, + { + "name": "pythainlp.corpus.icu._THAI_ICU_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/icu.py", + "line": 10 + }, + { + "name": "pythainlp.corpus.oscar.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/oscar.py", + "line": 12 + }, + { + "name": "pythainlp.corpus.oscar._OSCAR_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/oscar.py", + "line": 18 + }, + { + "name": "pythainlp.corpus.th_en_translit.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/th_en_translit.py", + "line": 17 + }, + { + "name": "pythainlp.corpus.th_en_translit._FILE_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/th_en_translit.py", + "line": 23 + }, + { + "name": "pythainlp.corpus.th_en_translit.TRANSLITERATE_EN", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/th_en_translit.py", + "line": 24 + }, + { + "name": "pythainlp.corpus.th_en_translit.TRANSLITERATE_FOLLOW_RTSG", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/th_en_translit.py", + "line": 25 + }, + { + "name": "pythainlp.corpus.th_en_translit.TRANSLITERATE_DICT", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/th_en_translit.py", + "line": 80 + }, + { + "name": "pythainlp.corpus.tnc.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/tnc.py", + "line": 8 + }, + { + "name": "pythainlp.corpus.tnc._UNIGRAM_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/tnc.py", + "line": 19 + }, + { + "name": "pythainlp.corpus.tnc._BIGRAM_CORPUS_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/tnc.py", + "line": 20 + }, + { + "name": "pythainlp.corpus.tnc._TRIGRAM_CORPUS_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/tnc.py", + "line": 21 + }, + { + "name": "pythainlp.corpus.ttc.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/ttc.py", + "line": 12 + }, + { + "name": "pythainlp.corpus.ttc._UNIGRAM_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/ttc.py", + "line": 18 + }, + { + "name": "pythainlp.corpus.volubilis._VOLUBILIS_WORDS", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/volubilis.py", + "line": 10 + }, + { + "name": "pythainlp.corpus.volubilis._VOLUBILIS_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/volubilis.py", + "line": 11 + }, + { + "name": "pythainlp.corpus.wikipedia._WIKIPEDIA_TITLES", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/wikipedia.py", + "line": 10 + }, + { + "name": "pythainlp.corpus.wikipedia._WIKIPEDIA_TITLES_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/corpus/wikipedia.py", + "line": 11 + }, + { + "name": "pythainlp.el.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/el/__init__.py", + "line": 6 + }, + { + "name": "pythainlp.generate.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/__init__.py", + "line": 6 + }, + { + "name": "pythainlp.generate.thai2fit.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/thai2fit.py", + "line": 12 + }, + { + "name": "pythainlp.generate.thai2fit.imdb", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/thai2fit.py", + "line": 41 + }, + { + "name": "pythainlp.generate.thai2fit.dummy_df", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/thai2fit.py", + "line": 42 + }, + { + "name": "pythainlp.generate.thai2fit.thwiki", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/thai2fit.py", + "line": 45 + }, + { + "name": "pythainlp.generate.thai2fit.thwiki_itos", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/thai2fit.py", + "line": 61 + }, + { + "name": "pythainlp.generate.thai2fit.thwiki_vocab", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/thai2fit.py", + "line": 62 + }, + { + "name": "pythainlp.generate.thai2fit.tt", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/thai2fit.py", + "line": 65 + }, + { + "name": "pythainlp.generate.thai2fit.processor", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/thai2fit.py", + "line": 71 + }, + { + "name": "pythainlp.generate.thai2fit.data_lm", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/thai2fit.py", + "line": 75 + }, + { + "name": "pythainlp.generate.thai2fit.config", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/thai2fit.py", + "line": 85 + }, + { + "name": "pythainlp.generate.thai2fit.trn_args", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/thai2fit.py", + "line": 99 + }, + { + "name": "pythainlp.generate.thai2fit.learn", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/thai2fit.py", + "line": 101 + }, + { + "name": "pythainlp.lm.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/lm/__init__.py", + "line": 5 + }, + { + "name": "pythainlp.morpheme.thaiwordcheck._TH_PREFIX_DIPHTHONG", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/morpheme/thaiwordcheck.py", + "line": 64 + }, + { + "name": "pythainlp.morpheme.thaiwordcheck._TH_CONSONANTS_PATTERN", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/morpheme/thaiwordcheck.py", + "line": 68 + }, + { + "name": "pythainlp.parse.core._tagger_name", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/parse/core.py", + "line": 9 + }, + { + "name": "pythainlp.phayathaibert.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/phayathaibert/__init__.py", + "line": 6 + }, + { + "name": "pythainlp.phayathaibert.core._PAT_URL", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/phayathaibert/core.py", + "line": 18 + }, + { + "name": "pythainlp.phayathaibert.core._model_name", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/phayathaibert/core.py", + "line": 20 + }, + { + "name": "pythainlp.phayathaibert.core._tokenizer", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/phayathaibert/core.py", + "line": 21 + }, + { + "name": "pythainlp.soundex.complete_soundex._complete_soundex_instance", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/soundex/complete_soundex.py", + "line": 616 + }, + { + "name": "pythainlp.soundex.sound._ft", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/soundex/sound.py", + "line": 12 + }, + { + "name": "pythainlp.soundex.sound._dst", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/soundex/sound.py", + "line": 13 + }, + { + "name": "pythainlp.spell.phunspell.pspell", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/phunspell.py", + "line": 22 + }, + { + "name": "pythainlp.spell.symspellpy._UNIGRAM_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/symspellpy.py", + "line": 28 + }, + { + "name": "pythainlp.spell.symspellpy._BIGRAM_CORPUS_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/symspellpy.py", + "line": 29 + }, + { + "name": "pythainlp.spell.symspellpy._sym_spell", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/symspellpy.py", + "line": 31 + }, + { + "name": "pythainlp.spell.symspellpy._unigram_file_ctx", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/symspellpy.py", + "line": 32 + }, + { + "name": "pythainlp.spell.symspellpy._load_lock", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/symspellpy.py", + "line": 35 + }, + { + "name": "pythainlp.spell.wanchanberta_thai_grammarly.use_cuda", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/wanchanberta_thai_grammarly.py", + "line": 27 + }, + { + "name": "pythainlp.spell.wanchanberta_thai_grammarly.device", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/wanchanberta_thai_grammarly.py", + "line": 28 + }, + { + "name": "pythainlp.spell.wanchanberta_thai_grammarly.tokenizer", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/wanchanberta_thai_grammarly.py", + "line": 29 + }, + { + "name": "pythainlp.spell.wanchanberta_thai_grammarly.tagging_model", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/wanchanberta_thai_grammarly.py", + "line": 56 + }, + { + "name": "pythainlp.spell.wanchanberta_thai_grammarly.tagging_model", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/wanchanberta_thai_grammarly.py", + "line": 58 + }, + { + "name": "pythainlp.spell.wanchanberta_thai_grammarly.ids_to_labels", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/wanchanberta_thai_grammarly.py", + "line": 59 + }, + { + "name": "pythainlp.spell.wanchanberta_thai_grammarly.mlm_model", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/wanchanberta_thai_grammarly.py", + "line": 100 + }, + { + "name": "pythainlp.spell.wanchanberta_thai_grammarly.mlm_model", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/wanchanberta_thai_grammarly.py", + "line": 104 + }, + { + "name": "pythainlp.spell.words_spelling_correction._WSC", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/words_spelling_correction.py", + "line": 251 + }, + { + "name": "pythainlp.summarize.freq._STOPWORDS", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/summarize/freq.py", + "line": 16 + }, + { + "name": "pythainlp.tag.thai_nner.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/thai_nner.py", + "line": 18 + }, + { + "name": "pythainlp.tag.thainer.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/thainer.py", + "line": 8 + }, + { + "name": "pythainlp.tag.thainer._TOKENIZER_ENGINE", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/thainer.py", + "line": 21 + }, + { + "name": "pythainlp.tokenize._utils._DIGITS_WITH_SEPARATOR", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/_utils.py", + "line": 11 + }, + { + "name": "pythainlp.tokenize.attacut._tokenizers_lock", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/attacut.py", + "line": 35 + }, + { + "name": "pythainlp.tokenize.budoux._parser", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/budoux.py", + "line": 18 + }, + { + "name": "pythainlp.tokenize.budoux._parser_lock", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/budoux.py", + "line": 19 + }, + { + "name": "pythainlp.tokenize.core._RE_WHITESPACE", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/core.py", + "line": 28 + }, + { + "name": "pythainlp.tokenize.core._RE_WORD_CHAR", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/core.py", + "line": 29 + }, + { + "name": "pythainlp.tokenize.crfcut._ENDERS", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/crfcut.py", + "line": 27 + }, + { + "name": "pythainlp.tokenize.crfcut._STARTERS", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/crfcut.py", + "line": 80 + }, + { + "name": "pythainlp.tokenize.crfcut._CRFCUT_DATA_FILENAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/crfcut.py", + "line": 177 + }, + { + "name": "pythainlp.tokenize.crfcut._tagger", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/crfcut.py", + "line": 178 + }, + { + "name": "pythainlp.tokenize.etcc._PAT_ENDING_CHAR", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/etcc.py", + "line": 37 + }, + { + "name": "pythainlp.tokenize.etcc._RE_ENDING_CHAR", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/etcc.py", + "line": 38 + }, + { + "name": "pythainlp.tokenize.han_solo._tagger", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/han_solo.py", + "line": 22 + }, + { + "name": "pythainlp.tokenize.han_solo._model_file_ctx", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/han_solo.py", + "line": 23 + }, + { + "name": "pythainlp.tokenize.han_solo._load_lock", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/han_solo.py", + "line": 24 + }, + { + "name": "pythainlp.tokenize.han_solo._to_feature", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/han_solo.py", + "line": 154 + }, + { + "name": "pythainlp.tokenize.longest._FRONT_DEP_CHAR", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/longest.py", + "line": 23 + }, + { + "name": "pythainlp.tokenize.longest._REAR_DEP_CHAR", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/longest.py", + "line": 39 + }, + { + "name": "pythainlp.tokenize.longest._TRAILING_CHAR", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/longest.py", + "line": 40 + }, + { + "name": "pythainlp.tokenize.longest._RE_NONTHAI", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/longest.py", + "line": 42 + }, + { + "name": "pythainlp.tokenize.longest._KNOWN", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/longest.py", + "line": 44 + }, + { + "name": "pythainlp.tokenize.longest._UNKNOWN", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/longest.py", + "line": 45 + }, + { + "name": "pythainlp.tokenize.longest._tokenizers_lock", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/longest.py", + "line": 163 + }, + { + "name": "pythainlp.tokenize.multi_cut._RE_NONTHAI", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/multi_cut.py", + "line": 51 + }, + { + "name": "pythainlp.tokenize.multi_cut._PAT_NONTHAI", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/multi_cut.py", + "line": 57 + }, + { + "name": "pythainlp.tokenize.nercut._thainer", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/nercut.py", + "line": 19 + }, + { + "name": "pythainlp.tokenize.newmm._PAT_NONTHAI", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/newmm.py", + "line": 32 + }, + { + "name": "pythainlp.tokenize.newmm._PAT_THAI_TWOCHARS", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/newmm.py", + "line": 43 + }, + { + "name": "pythainlp.tokenize.newmm._MAX_GRAPH_SIZE", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/newmm.py", + "line": 47 + }, + { + "name": "pythainlp.tokenize.newmm._TEXT_SCAN_POINT", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/newmm.py", + "line": 50 + }, + { + "name": "pythainlp.tokenize.newmm._TEXT_SCAN_LEFT", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/newmm.py", + "line": 51 + }, + { + "name": "pythainlp.tokenize.newmm._TEXT_SCAN_RIGHT", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/newmm.py", + "line": 52 + }, + { + "name": "pythainlp.tokenize.newmm._TEXT_SCAN_BEGIN", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/newmm.py", + "line": 53 + }, + { + "name": "pythainlp.tokenize.newmm._TEXT_SCAN_END", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/newmm.py", + "line": 54 + }, + { + "name": "pythainlp.tokenize.nlpo3._NLPO3_DEFAULT_DICT_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/nlpo3.py", + "line": 17 + }, + { + "name": "pythainlp.tokenize.nlpo3._NLPO3_DEFAULT_DICT", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/nlpo3.py", + "line": 18 + }, + { + "name": "pythainlp.tokenize.nlpo3._dict_file_ctx", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/nlpo3.py", + "line": 19 + }, + { + "name": "pythainlp.tokenize.nlpo3._load_lock", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/nlpo3.py", + "line": 20 + }, + { + "name": "pythainlp.tokenize.oskut._DEFAULT_ENGINE", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/oskut.py", + "line": 19 + }, + { + "name": "pythainlp.tokenize.oskut._engine_lock", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/oskut.py", + "line": 20 + }, + { + "name": "pythainlp.tokenizeicu._thread_local", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/pyicu.py", + "line": 21 + }, + { + "name": "pythainlp.tokenize.sefr_cut._DEFAULT_ENGINE", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/sefr_cut.py", + "line": 18 + }, + { + "name": "pythainlp.tokenize.sefr_cut._engine_lock", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/sefr_cut.py", + "line": 19 + }, + { + "name": "pythainlp.tokenize.tcc._RE_TCC", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/tcc.py", + "line": 20 + }, + { + "name": "pythainlp.tokenize.tcc._PAT_TCC", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/tcc.py", + "line": 59 + }, + { + "name": "pythainlp.tokenize.tcc_p._RE_TCC", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/tcc_p.py", + "line": 21 + }, + { + "name": "pythainlp.tokenize.tcc_p._PAT_TCC", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/tcc_p.py", + "line": 59 + }, + { + "name": "pythainlp.tokenize.wtsplit._MODEL", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/wtsplit.py", + "line": 16 + }, + { + "name": "pythainlp.tokenize.wtsplit._MODEL_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/wtsplit.py", + "line": 17 + }, + { + "name": "pythainlp.tokenize.wtsplit._model_lock", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/wtsplit.py", + "line": 18 + }, + { + "name": "pythainlp.tools.misspell.THAI_CHARACTERS_WITHOUT_SHIFT", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tools/misspell.py", + "line": 10 + }, + { + "name": "pythainlp.tools.misspell.THAI_CHARACTERS_WITH_SHIFT", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tools/misspell.py", + "line": 17 + }, + { + "name": "pythainlp.tools.misspell.ENGLISH_CHARACTERS_WITHOUT_SHIFT", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tools/misspell.py", + "line": 24 + }, + { + "name": "pythainlp.tools.misspell.ENGLISH_CHARACTERS_WITH_SHIFT", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tools/misspell.py", + "line": 31 + }, + { + "name": "pythainlp.tools.misspell.ALL_CHARACTERS", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tools/misspell.py", + "line": 39 + }, + { + "name": "pythainlp.tools.path.PYTHAINLP_DEFAULT_DATA_DIR", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tools/path.py", + "line": 21 + }, + { + "name": "pythainlp.translate.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/__init__.py", + "line": 6 + }, + { + "name": "pythainlp.translate.en_th._EN_TH_MODEL_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/en_th.py", + "line": 32 + }, + { + "name": "pythainlp.translate.en_th._EN_TH_FILE_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/en_th.py", + "line": 34 + }, + { + "name": "pythainlp.translate.en_th._TH_EN_MODEL_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/en_th.py", + "line": 36 + }, + { + "name": "pythainlp.translate.en_th._TH_EN_FILE_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/en_th.py", + "line": 38 + }, + { + "name": "pythainlp.translate.tokenization_small100.SPIECE_UNDERLINE", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 35 + }, + { + "name": "pythainlp.translate.tokenization_small100.VOCAB_FILES_NAMES", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 37 + }, + { + "name": "pythainlp.translate.tokenization_small100.PRETRAINED_VOCAB_FILES_MAP", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 43 + }, + { + "name": "pythainlp.translate.tokenization_small100.PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 55 + }, + { + "name": "pythainlp.translate.tokenization_small100.FAIRSEQ_LANGUAGE_CODES", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/tokenization_small100.py", + "line": 60 + }, + { + "name": "pythainlp.translate.word2word_translate.support_list", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/word2word_translate.py", + "line": 10 + }, + { + "name": "pythainlp.transliterate.iso_11940._all_dict", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/iso_11940.py", + "line": 125 + }, + { + "name": "pythainlp.transliterate.iso_11940._keys_set", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/iso_11940.py", + "line": 131 + }, + { + "name": "pythainlp.transliterate.lookup._TRANSLITERATE_IDX", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/lookup.py", + "line": 22 + }, + { + "name": "pythainlp.transliterateicu._ICU_THAI_TO_LATIN", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/pyicu.py", + "line": 16 + }, + { + "name": "pythainlp.transliterate.royin._vowel_patterns", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/royin.py", + "line": 73 + }, + { + "name": "pythainlp.transliterate.royin._vowel_patterns", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/royin.py", + "line": 74 + }, + { + "name": "pythainlp.transliterate.royin._vowel_patterns", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/royin.py", + "line": 75 + }, + { + "name": "pythainlp.transliterate.spoonerism._list_consonants", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/spoonerism.py", + "line": 9 + }, + { + "name": "pythainlp.transliterate.thai2rom.device", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 20 + }, + { + "name": "pythainlp.transliterate.thai2rom._MODEL_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 22 + }, + { + "name": "pythainlp.transliterate.thai2rom._THAI_TO_ROM", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 398 + }, + { + "name": "pythainlp.transliterate.thai2rom_onnx._MODEL_ENCODER_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", + "line": 19 + }, + { + "name": "pythainlp.transliterate.thai2rom_onnx._MODEL_DECODER_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", + "line": 20 + }, + { + "name": "pythainlp.transliterate.thai2rom_onnx._MODEL_CONFIG_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", + "line": 21 + }, + { + "name": "pythainlp.transliterate.thai2rom_onnx._THAI_TO_ROM_ONNX", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom_onnx.py", + "line": 184 + }, + { + "name": "pythainlp.transliterate.thaig2p.device", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 23 + }, + { + "name": "pythainlp.transliterate.thaig2p._MODEL_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 25 + }, + { + "name": "pythainlp.transliterate.thaig2p._THAI_G2P", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 413 + }, + { + "name": "pythainlp.transliterate.thaig2p_v2._THAI_G2P", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p_v2.py", + "line": 45 + }, + { + "name": "pythainlp.transliterate.umt5_thaig2p._THAI_G2P", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/umt5_thaig2p.py", + "line": 45 + }, + { + "name": "pythainlp.transliterate.w2p._GRAPHEMES", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", + "line": 19 + }, + { + "name": "pythainlp.transliterate.w2p._PHONEMES", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", + "line": 22 + }, + { + "name": "pythainlp.transliterate.w2p._MODEL_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", + "line": 26 + }, + { + "name": "pythainlp.transliterate.w2p.hp", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", + "line": 41 + }, + { + "name": "pythainlp.transliterate.w2p._THAI_W2P", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", + "line": 229 + }, + { + "name": "pythainlp.ulmfit.core.device", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/core.py", + "line": 34 + }, + { + "name": "pythainlp.ulmfit.core._MODEL_NAME_LSTM", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/core.py", + "line": 36 + }, + { + "name": "pythainlp.ulmfit.core._ITOS_NAME_LSTM", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/core.py", + "line": 37 + }, + { + "name": "pythainlp.ulmfit.core.THWIKI_LSTM", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/core.py", + "line": 44 + }, + { + "name": "pythainlp.ulmfit.core.pre_rules_th", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/core.py", + "line": 76 + }, + { + "name": "pythainlp.ulmfit.core.post_rules_th", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/core.py", + "line": 86 + }, + { + "name": "pythainlp.ulmfit.core.pre_rules_th_sparse", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/core.py", + "line": 89 + }, + { + "name": "pythainlp.ulmfit.core.post_rules_th_sparse", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/core.py", + "line": 90 + }, + { + "name": "pythainlp.ulmfit.preprocess._TK_UNK", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/preprocess.py", + "line": 15 + }, + { + "name": "pythainlp.ulmfit.preprocess._TK_REP", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/preprocess.py", + "line": 16 + }, + { + "name": "pythainlp.ulmfit.preprocess._TK_WREP", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/preprocess.py", + "line": 17 + }, + { + "name": "pythainlp.ulmfit.preprocess._TK_END", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/preprocess.py", + "line": 18 + }, + { + "name": "pythainlp.ulmfit.preprocess._TK_URL", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ulmfit/preprocess.py", + "line": 19 + }, + { + "name": "pythainlp.util.collate._RE_TONE", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/collate.py", + "line": 13 + }, + { + "name": "pythainlp.util.collate._RE_LV_C", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/collate.py", + "line": 14 + }, + { + "name": "pythainlp.util.date.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py", + "line": 17 + }, + { + "name": "pythainlp.util.date.thai_abbr_weekdays", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py", + "line": 31 + }, + { + "name": "pythainlp.util.date.thai_full_weekdays", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py", + "line": 32 + }, + { + "name": "pythainlp.util.date.thai_abbr_months", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py", + "line": 42 + }, + { + "name": "pythainlp.util.date.thai_full_months", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py", + "line": 56 + }, + { + "name": "pythainlp.util.date.thai_full_month_lists", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py", + "line": 70 + }, + { + "name": "pythainlp.util.date.thai_full_month_lists_regex", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py", + "line": 84 + }, + { + "name": "pythainlp.util.date.year_all_regex", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py", + "line": 87 + }, + { + "name": "pythainlp.util.date.dates_list", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py", + "line": 88 + }, + { + "name": "pythainlp.util.date._DAY", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/date.py", + "line": 96 + }, + { + "name": "pythainlp.util.digitconv._spell_digit", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/digitconv.py", + "line": 47 + }, + { + "name": "pythainlp.util.digitconv._arabic_thai_translate_table", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/digitconv.py", + "line": 60 + }, + { + "name": "pythainlp.util.digitconv._thai_arabic_translate_table", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/digitconv.py", + "line": 61 + }, + { + "name": "pythainlp.util.digitconv._digit_spell_translate_table", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/digitconv.py", + "line": 62 + }, + { + "name": "pythainlp.util.emojiconv._th_emoji", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/emojiconv.py", + "line": 1829 + }, + { + "name": "pythainlp.util.emojiconv._emojis", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/emojiconv.py", + "line": 1831 + }, + { + "name": "pythainlp.util.emojiconv._emoji_regex", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/emojiconv.py", + "line": 1832 + }, + { + "name": "pythainlp.util.emojiconv._delimiter", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/emojiconv.py", + "line": 1833 + }, + { + "name": "pythainlp.util.keyboard.EN_TH_KEYB_PAIRS", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/keyboard.py", + "line": 10 + }, + { + "name": "pythainlp.util.keyboard.TH_EN_KEYB_PAIRS", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/keyboard.py", + "line": 105 + }, + { + "name": "pythainlp.util.keyboard.EN_TH_TRANSLATE_TABLE", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/keyboard.py", + "line": 107 + }, + { + "name": "pythainlp.util.keyboard.TH_EN_TRANSLATE_TABLE", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/keyboard.py", + "line": 108 + }, + { + "name": "pythainlp.util.keyboard.TIS_820_2531_MOD", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/keyboard.py", + "line": 110 + }, + { + "name": "pythainlp.util.keyboard.TIS_820_2531_MOD_SHIFT", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/keyboard.py", + "line": 116 + }, + { + "name": "pythainlp.util.keywords._STOPWORDS", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/keywords.py", + "line": 11 + }, + { + "name": "pythainlp.util.morse.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/morse.py", + "line": 126 + }, + { + "name": "pythainlp.util.morse.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/morse.py", + "line": 130 + }, + { + "name": "pythainlp.util.morse.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/morse.py", + "line": 133 + }, + { + "name": "pythainlp.util.normalize._RE_TONEMARKS", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/normalize.py", + "line": 51 + }, + { + "name": "pythainlp.util.normalize._RE_REMOVE_NEWLINES", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/normalize.py", + "line": 53 + }, + { + "name": "pythainlp.util.normalize._RE_REMOVE_SPACES_BEFORE_NONBASE", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/normalize.py", + "line": 58 + }, + { + "name": "pythainlp.util.phoneme.consonants_ipa_nectec", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/phoneme.py", + "line": 14 + }, + { + "name": "pythainlp.util.phoneme.monophthong_ipa_nectec", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/phoneme.py", + "line": 39 + }, + { + "name": "pythainlp.util.phoneme.diphthong_ipa_nectec", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/phoneme.py", + "line": 62 + }, + { + "name": "pythainlp.util.phoneme.tones_ipa_nectec", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/phoneme.py", + "line": 71 + }, + { + "name": "pythainlp.util.phoneme.dict_nectec_to_ipa", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/phoneme.py", + "line": 79 + }, + { + "name": "pythainlp.util.phoneme.dict_ipa_rtgs", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/phoneme.py", + "line": 124 + }, + { + "name": "pythainlp.util.phoneme.dict_ipa_rtgs_final", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/phoneme.py", + "line": 194 + }, + { + "name": "pythainlp.util.pronounce.kv", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/pronounce.py", + "line": 14 + }, + { + "name": "pythainlp.util.pronounce.all_thai_words_dict", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/pronounce.py", + "line": 15 + }, + { + "name": "pythainlp.util.pronounce.thai_vowel", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/pronounce.py", + "line": 45 + }, + { + "name": "pythainlp.util.pronounce.thai_vowel_all", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/pronounce.py", + "line": 51 + }, + { + "name": "pythainlp.util.spell_words._r1", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/spell_words.py", + "line": 21 + }, + { + "name": "pythainlp.util.spell_words._r2", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/spell_words.py", + "line": 22 + }, + { + "name": "pythainlp.util.spell_words.tonemarks", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/spell_words.py", + "line": 23 + }, + { + "name": "pythainlp.util.spell_words.dict_vowel_ex", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/spell_words.py", + "line": 38 + }, + { + "name": "pythainlp.util.spell_words.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/spell_words.py", + "line": 40 + }, + { + "name": "pythainlp.util.spell_words.dict_vowel", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/spell_words.py", + "line": 43 + }, + { + "name": "pythainlp.util.spell_words.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/spell_words.py", + "line": 45 + }, + { + "name": "pythainlp.util.spell_words.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/spell_words.py", + "line": 49 + }, + { + "name": "pythainlp.util.spell_words.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/spell_words.py", + "line": 51 + }, + { + "name": "pythainlp.util.spell_words.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/spell_words.py", + "line": 53 + }, + { + "name": "pythainlp.util.spell_words.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/spell_words.py", + "line": 55 + }, + { + "name": "pythainlp.util.strftime.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/strftime.py", + "line": 20 + }, + { + "name": "pythainlp.util.strftime._HA_TH_DIGITS", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/strftime.py", + "line": 24 + }, + { + "name": "pythainlp.util.strftime._BE_AD_DIFFERENCE", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/strftime.py", + "line": 25 + }, + { + "name": "pythainlp.util.strftime._NEED_L10N", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/strftime.py", + "line": 27 + }, + { + "name": "pythainlp.util.strftime._EXTENSIONS", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/strftime.py", + "line": 28 + }, + { + "name": "pythainlp.util.syllable.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/syllable.py", + "line": 61 + }, + { + "name": "pythainlp.util.thai_lunar_date._BEGIN_DATES", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/thai_lunar_date.py", + "line": 129 + }, + { + "name": "pythainlp.util.thai_lunar_date._DAYS_354", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/thai_lunar_date.py", + "line": 188 + }, + { + "name": "pythainlp.util.thai_lunar_date._DAYS_355", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/thai_lunar_date.py", + "line": 189 + }, + { + "name": "pythainlp.util.thai_lunar_date._DAYS_384", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/thai_lunar_date.py", + "line": 190 + }, + { + "name": "pythainlp.util.time._TIME_FORMAT_WITH_SEC", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/time.py", + "line": 19 + }, + { + "name": "pythainlp.util.time._TIME_FORMAT_WITHOUT_SEC", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/time.py", + "line": 20 + }, + { + "name": "pythainlp.util.time._DICT_THAI_TIME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/time.py", + "line": 21 + }, + { + "name": "pythainlp.util.time._THAI_TIME_AFFIX", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/time.py", + "line": 56 + }, + { + "name": "pythainlp.wangchanberta.__all__", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/__init__.py", + "line": 4 + }, + { + "name": "pythainlp.wangchanberta.core._model_name", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py", + "line": 12 + }, + { + "name": "pythainlp.wangchanberta.core._tokenizer", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py", + "line": 13 + }, + { + "name": "pythainlp.word_vector.core.WV_DIM", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/word_vector/core.py", + "line": 15 + }, + { + "name": "pythainlp.word_vector.core._MODEL_NAME", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/word_vector/core.py", + "line": 17 + }, + { + "name": "pythainlp.word_vector.core._TK_SP", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/word_vector/core.py", + "line": 19 + }, + { + "name": "pythainlp.word_vector.core._TK_EOL", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/word_vector/core.py", + "line": 20 + }, + { + "name": "pythainlp.wsd.core._wsd_dict", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py", + "line": 12 + }, + { + "name": "pythainlp.wsd.core._mean_all", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py", + "line": 13 + }, + { + "name": "pythainlp.wsd.core.unknown", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py", + "line": 16 + }, + { + "name": "pythainlp.wsd.core._all_word", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py", + "line": 18 + }, + { + "name": "pythainlp.wsd.core._TRIE", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py", + "line": 19 + }, + { + "name": "pythainlp.wsd.core._word_cut", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py", + "line": 20 + }, + { + "name": "pythainlp.wsd.core._MODEL", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wsd/core.py", + "line": 22 + } + ], + "type_aliases": [] } \ No newline at end of file diff --git a/pythainlp/__init__.py b/pythainlp/__init__.py index 36afd5c5f..40659b52e 100644 --- a/pythainlp/__init__.py +++ b/pythainlp/__init__.py @@ -1,24 +1,24 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -__version__ = "5.2.0" +__version__: str = "5.2.0" -thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ" # 44 chars +thai_consonants: str = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ" # 44 chars -thai_vowels = ( +thai_vowels: str = ( "\u0e24\u0e26\u0e30\u0e31\u0e32\u0e33\u0e34\u0e35\u0e36\u0e37" + "\u0e38\u0e39\u0e40\u0e41\u0e42\u0e43\u0e44\u0e45\u0e4d\u0e47" ) # 20 -thai_lead_vowels = "\u0e40\u0e41\u0e42\u0e43\u0e44" # 5 -thai_follow_vowels = "\u0e30\u0e32\u0e33\u0e45" # 4 -thai_above_vowels = "\u0e31\u0e34\u0e35\u0e36\u0e37\u0e4d\u0e47" # 7 -thai_below_vowels = "\u0e38\u0e39" # 2 +thai_lead_vowels: str = "\u0e40\u0e41\u0e42\u0e43\u0e44" # 5 +thai_follow_vowels: str = "\u0e30\u0e32\u0e33\u0e45" # 4 +thai_above_vowels: str = "\u0e31\u0e34\u0e35\u0e36\u0e37\u0e4d\u0e47" # 7 +thai_below_vowels: str = "\u0e38\u0e39" # 2 thai_tonemarks: str = "\u0e48\u0e49\u0e4a\u0e4b" # 4 # Paiyannoi, Maiyamok, Phinthu, Thanthakhat, Nikhahit, Yamakkan: # These signs can be part of a word -thai_signs = "\u0e2f\u0e3a\u0e46\u0e4c\u0e4d\u0e4e" # 6 chars +thai_signs: str = "\u0e2f\u0e3a\u0e46\u0e4c\u0e4d\u0e4e" # 6 chars # Any Thai character that can be part of a word thai_letters: str = "".join( @@ -27,19 +27,19 @@ # Fongman, Angkhankhu, Khomut: # These characters are section markers -thai_punctuations = "\u0e4f\u0e5a\u0e5b" # 3 chars +thai_punctuations: str = "\u0e4f\u0e5a\u0e5b" # 3 chars -thai_digits = "๐๑๒๓๔๕๖๗๘๙" # 10 -thai_symbols = "\u0e3f" # Thai Bath ฿ +thai_digits: str = "๐๑๒๓๔๕๖๗๘๙" # 10 +thai_symbols: str = "\u0e3f" # Thai Bath ฿ # All Thai characters that are presented in Unicode -thai_characters = "".join( +thai_characters: str = "".join( [thai_letters, thai_punctuations, thai_digits, thai_symbols] ) # Thai pangram by Sungsit Sawaiwan # CC BY-SA License # Source: https://fontuni.com/articles/2015-07-12-thai-poetgram.html -thai_pangram = """กีฬาบังลังก์ ฿๑,๒๓๔,๕๖๗,๘๙๐ +thai_pangram: str = """กีฬาบังลังก์ ฿๑,๒๓๔,๕๖๗,๘๙๐ ๏ จับฅอคนบั่นต้อง อาญา ขุดฆ่าโคตรฃัตติยา ซ่านม้วย ธรรมฤๅผ่อนรักษา ใจชั่ว โฉดแฮ @@ -50,7 +50,7 @@ โกรธจี๊ดจ๋อยจ่มถ้ำ อยู่เฝ้า “อตฺตา” ๚ะ๛ ๑๒ กรกฎาคม ๒๕๕๘""" -__all__ = [ +__all__: list[str] = [ "collate", "correct", "pos_tag", diff --git a/pythainlp/__main__.py b/pythainlp/__main__.py index 91a280366..c91fc8953 100644 --- a/pythainlp/__main__.py +++ b/pythainlp/__main__.py @@ -1,13 +1,16 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import argparse import sys +from typing import Optional from pythainlp import cli -def main(argv=None): +def main(argv: Optional[list[str]] = None) -> None: """ThaiNLP command line.""" if not argv: argv = sys.argv diff --git a/pythainlp/ancient/__init__.py b/pythainlp/ancient/__init__.py index 660baac1c..ad88080ea 100644 --- a/pythainlp/ancient/__init__.py +++ b/pythainlp/ancient/__init__.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 """Ancient versions of the Thai language""" -__all__ = ["aksonhan_to_current", "convert_currency"] +__all__: list[str] = ["aksonhan_to_current", "convert_currency"] from pythainlp.ancient.aksonhan import aksonhan_to_current from pythainlp.ancient.currency import convert_currency diff --git a/pythainlp/augment/__init__.py b/pythainlp/augment/__init__.py index 5fb975bf4..2d7872502 100644 --- a/pythainlp/augment/__init__.py +++ b/pythainlp/augment/__init__.py @@ -3,6 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 """Thai text augment""" -__all__ = ["WordNetAug"] +__all__: list[str] = ["WordNetAug"] from pythainlp.augment.wordnet import WordNetAug diff --git a/pythainlp/augment/lm/__init__.py b/pythainlp/augment/lm/__init__.py index 40b3c32bc..d2919e986 100644 --- a/pythainlp/augment/lm/__init__.py +++ b/pythainlp/augment/lm/__init__.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 """Language Models""" -__all__ = [ +__all__: list[str] = [ "FastTextAug", "Thai2transformersAug", "ThaiTextAugmenter", diff --git a/pythainlp/augment/lm/fasttext.py b/pythainlp/augment/lm/fasttext.py index 8ca099c01..6c5dad320 100644 --- a/pythainlp/augment/lm/fasttext.py +++ b/pythainlp/augment/lm/fasttext.py @@ -4,6 +4,11 @@ from __future__ import annotations import itertools +from typing import TYPE_CHECKING, Union + +if TYPE_CHECKING: + from gensim.models import FastText + from gensim.models.keyedvectors import KeyedVectors from pythainlp.tokenize import word_tokenize @@ -14,18 +19,23 @@ class FastTextAug: :param str model_path: path of model file """ - def __init__(self, model_path: str): + model: Union[FastText, KeyedVectors] + dict_wv: list[str] + sentence: list[str] + list_synonym: list[list[str]] + + def __init__(self, model_path: str) -> None: """:param str model_path: path of model file""" from gensim.models.fasttext import FastText as FastText_gensim from gensim.models.keyedvectors import KeyedVectors if model_path.endswith(".bin"): - self.model = FastText_gensim.load_facebook_vectors(model_path) + self.model: Union[FastText, KeyedVectors] = FastText_gensim.load_facebook_vectors(model_path) elif model_path.endswith(".vec"): self.model = KeyedVectors.load_word2vec_format(model_path) else: self.model = FastText_gensim.load(model_path) - self.dict_wv = list(self.model.key_to_index.keys()) + self.dict_wv: list[str] = list(self.model.key_to_index.keys()) def tokenize(self, text: str) -> list[str]: """Thai text tokenization for fastText @@ -69,8 +79,8 @@ def augment( :return: list of synonyms :rtype: List[Tuple[str]] """ - self.sentence = self.tokenize(sentence) - self.list_synonym = self.modify_sent(self.sentence, p=p) + self.sentence: list[str] = self.tokenize(sentence) + self.list_synonym: list[list[str]] = self.modify_sent(self.sentence, p=p) new_sentences = [] for x in list(itertools.product(*self.list_synonym))[0:n_sent]: new_sentences.append(x) diff --git a/pythainlp/augment/lm/phayathaibert.py b/pythainlp/augment/lm/phayathaibert.py index cc0312312..eada2f975 100644 --- a/pythainlp/augment/lm/phayathaibert.py +++ b/pythainlp/augment/lm/phayathaibert.py @@ -5,6 +5,10 @@ import random import re +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from transformers import AutoModelForMaskedLM, AutoTokenizer, Pipeline from pythainlp.phayathaibert.core import ThaiTextProcessor @@ -12,6 +16,11 @@ class ThaiTextAugmenter: + tokenizer: "AutoTokenizer" + model_for_masked_lm: "AutoModelForMaskedLM" + model: "Pipeline" + processor: "ThaiTextProcessor" + def __init__(self) -> None: from transformers import ( AutoModelForMaskedLM, @@ -19,8 +28,8 @@ def __init__(self) -> None: pipeline, ) - self.tokenizer = AutoTokenizer.from_pretrained(_MODEL_NAME) - self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained( + self.tokenizer = AutoTokenizer.from_pretrained(_MODEL_NAME) # type: ignore[assignment] + self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained( # type: ignore[assignment] _MODEL_NAME ) self.model = pipeline( diff --git a/pythainlp/augment/lm/wangchanberta.py b/pythainlp/augment/lm/wangchanberta.py index aef83400a..49ee6ecd2 100644 --- a/pythainlp/augment/lm/wangchanberta.py +++ b/pythainlp/augment/lm/wangchanberta.py @@ -3,11 +3,23 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from transformers import CamembertTokenizer, Pipeline + model_name = "airesearch/wangchanberta-base-att-spm-uncased" class Thai2transformersAug: - def __init__(self): + model_name: str + target_tokenizer: type["CamembertTokenizer"] + tokenizer: "CamembertTokenizer" + fill_mask: "Pipeline" + MASK_TOKEN: str + input_text: str + + def __init__(self) -> None: from transformers import ( CamembertTokenizer, pipeline, @@ -15,7 +27,7 @@ def __init__(self): self.model_name = "airesearch/wangchanberta-base-att-spm-uncased" self.target_tokenizer = CamembertTokenizer - self.tokenizer = CamembertTokenizer.from_pretrained( + self.tokenizer = CamembertTokenizer.from_pretrained( # type: ignore[assignment] self.model_name, revision="main" ) self.tokenizer.additional_special_tokens = [ @@ -31,7 +43,9 @@ def __init__(self): ) self.MASK_TOKEN = self.tokenizer.mask_token - def generate(self, sentence: str, num_replace_tokens: int = 3): + def generate( + self, sentence: str, num_replace_tokens: int = 3 + ) -> list[str]: sent2: list[str] = [] self.input_text = sentence sent = [ @@ -74,4 +88,4 @@ def augment(self, sentence: str, num_replace_tokens: int = 3) -> list[str]: 'ช้างมีทั้งหมด 50 ตัว บนหัว'] """ sent2 = self.generate(sentence, num_replace_tokens) - return sent2 # type: ignore[no-any-return] + return sent2 diff --git a/pythainlp/augment/word2vec/__init__.py b/pythainlp/augment/word2vec/__init__.py index 1b27b47da..1e997d6be 100644 --- a/pythainlp/augment/word2vec/__init__.py +++ b/pythainlp/augment/word2vec/__init__.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 """Word2Vec""" -__all__ = ["Word2VecAug", "Thai2fitAug", "LTW2VAug"] +__all__: list[str] = ["Word2VecAug", "Thai2fitAug", "LTW2VAug"] from pythainlp.augment.word2vec.core import Word2VecAug from pythainlp.augment.word2vec.ltw2v import LTW2VAug diff --git a/pythainlp/augment/word2vec/bpemb_wv.py b/pythainlp/augment/word2vec/bpemb_wv.py index 510551602..5e43bbdee 100644 --- a/pythainlp/augment/word2vec/bpemb_wv.py +++ b/pythainlp/augment/word2vec/bpemb_wv.py @@ -3,8 +3,14 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from typing import TYPE_CHECKING, Any + from pythainlp.augment.word2vec.core import Word2VecAug +if TYPE_CHECKING: + from bpemb import BPEmb + from gensim.models.keyedvectors import KeyedVectors + class BPEmbAug: """Thai Text Augment using word2vec from BPEmb @@ -13,7 +19,17 @@ class BPEmbAug: `github.com/bheinzerling/bpemb `_ """ - def __init__(self, lang: str = "th", vs: int = 100000, dim: int = 300): + bpemb_temp: BPEmb + model: KeyedVectors + aug: Word2VecAug + sentence: str + temp: list[tuple[str, ...]] + temp_new: list[str] + t: str + + def __init__( + self, lang: str = "th", vs: int = 100000, dim: int = 300 + ) -> None: from bpemb import BPEmb self.bpemb_temp = BPEmb(lang=lang, dim=dim, vs=vs) @@ -26,7 +42,7 @@ def tokenizer(self, text: str) -> list[str]: """ return self.bpemb_temp.encode(text) # type: ignore[no-any-return] - def load_w2v(self): + def load_w2v(self) -> None: """Load BPEmb model""" self.aug = Word2VecAug( self.model, tokenize=self.tokenizer, type="model" diff --git a/pythainlp/augment/word2vec/core.py b/pythainlp/augment/word2vec/core.py index 640703a56..2a177625b 100644 --- a/pythainlp/augment/word2vec/core.py +++ b/pythainlp/augment/word2vec/core.py @@ -4,10 +4,17 @@ from __future__ import annotations import itertools -from typing import Callable +from typing import TYPE_CHECKING, Any, Callable + +if TYPE_CHECKING: + from gensim.models.keyedvectors import KeyedVectors class Word2VecAug: + tokenizer: Callable[[str], list[str]] + model: "KeyedVectors" + dict_wv: list[str] + def __init__( self, model: str, @@ -20,16 +27,16 @@ def __init__( """ import gensim.models.keyedvectors as word2vec - self.tokenizer = tokenize + self.tokenizer: Callable[[str], list[str]] = tokenize if type == "file": - self.model = word2vec.KeyedVectors.load_word2vec_format(model) + self.model: "KeyedVectors" = word2vec.KeyedVectors.load_word2vec_format(model) elif type == "binary": self.model = word2vec.KeyedVectors.load_word2vec_format( model, binary=True, unicode_errors="ignore" ) else: self.model = model - self.dict_wv = list(self.model.key_to_index.keys()) + self.dict_wv: list[str] = list(self.model.key_to_index.keys()) def modify_sent(self, sent: list[str], p: float = 0.7) -> list[list[str]]: """:param list[str] sent: list of tokens diff --git a/pythainlp/augment/word2vec/ltw2v.py b/pythainlp/augment/word2vec/ltw2v.py index 34d1a8de2..8f7d8ebe8 100644 --- a/pythainlp/augment/word2vec/ltw2v.py +++ b/pythainlp/augment/word2vec/ltw2v.py @@ -3,7 +3,15 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from pythainlp.augment.word2vec.core import Word2VecAug +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pythainlp.augment.word2vec.core import Word2VecAug + +from pythainlp.augment.word2vec.core import Word2VecAug as _Word2VecAug + +# Make it accessible for runtime +Word2VecAug = _Word2VecAug from pythainlp.corpus import get_corpus_path from pythainlp.tokenize import word_tokenize @@ -15,7 +23,10 @@ class LTW2VAug: `github.com/PyThaiNLP/large-thaiword2vec `_ """ - def __init__(self): + ltw2v_wv: str | None + aug: Word2VecAug + + def __init__(self) -> None: self.ltw2v_wv = get_corpus_path("ltw2v") self.load_w2v() @@ -25,7 +36,7 @@ def tokenizer(self, text: str) -> list[str]: """ return word_tokenize(text, engine="newmm") - def load_w2v(self): # insert substitute + def load_w2v(self) -> None: # insert substitute """Load LTW2V's word2vec model""" if self.ltw2v_wv is None: raise ValueError( diff --git a/pythainlp/augment/word2vec/thai2fit.py b/pythainlp/augment/word2vec/thai2fit.py index d24ae10a2..18c54b706 100644 --- a/pythainlp/augment/word2vec/thai2fit.py +++ b/pythainlp/augment/word2vec/thai2fit.py @@ -3,7 +3,15 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from pythainlp.augment.word2vec.core import Word2VecAug +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pythainlp.augment.word2vec.core import Word2VecAug + +from pythainlp.augment.word2vec.core import Word2VecAug as _Word2VecAug + +# Make it accessible for runtime +Word2VecAug = _Word2VecAug from pythainlp.corpus import get_corpus_path from pythainlp.tokenize import thai2fit_tokenizer @@ -15,7 +23,10 @@ class Thai2fitAug: `github.com/cstorm125/thai2fit `_ """ - def __init__(self): + thai2fit_wv: str | None + aug: Word2VecAug + + def __init__(self) -> None: self.thai2fit_wv = get_corpus_path("thai2fit_wv") self.load_w2v() @@ -26,7 +37,7 @@ def tokenizer(self, text: str) -> list[str]: tok = thai2fit_tokenizer() return tok.word_tokenize(text) # type: ignore[no-any-return] - def load_w2v(self): + def load_w2v(self) -> None: """Load Thai2Fit's word2vec model""" if self.thai2fit_wv is None: raise ValueError( diff --git a/pythainlp/augment/wordnet.py b/pythainlp/augment/wordnet.py index 309ab66f0..740d3566f 100644 --- a/pythainlp/augment/wordnet.py +++ b/pythainlp/augment/wordnet.py @@ -101,7 +101,7 @@ } -def postype2wordnet(pos: str, corpus: str): +def postype2wordnet(pos: str, corpus: str) -> Optional[str]: """Convert part-of-speech type to wordnet type :param str pos: POS type @@ -112,13 +112,13 @@ def postype2wordnet(pos: str, corpus: str): """ if corpus not in ["orchid"]: return None - return orchid[pos] + return orchid[pos] # type: ignore[no-any-return] class WordNetAug: """Text Augment using wordnet""" - def __init__(self): + def __init__(self) -> None: pass def find_synonyms( @@ -135,7 +135,7 @@ def find_synonyms( :return: list of synonyms :rtype: list[str] """ - self.synonyms = [] + self.synonyms: list[str] = [] if pos is None: self.list_synsets = wordnet.synsets(word) else: @@ -149,7 +149,7 @@ def find_synonyms( for self.syn in self.synset.lemma_names(lang="tha"): self.synonyms.append(self.syn) - self.synonyms_without_duplicates = list( + self.synonyms_without_duplicates: list[str] = list( OrderedDict.fromkeys(self.synonyms) ) return self.synonyms_without_duplicates @@ -188,13 +188,13 @@ def augment( ('เรา', 'ชอบ', 'ไปยัง', 'รร.')] """ new_sentences = [] - self.list_words = tokenize(sentence) - self.list_synonym = [] - self.p_all = 1 + self.list_words: list[str] = tokenize(sentence) + self.list_synonym: list[list[str]] = [] + self.p_all: int = 1 if postag: - self.list_pos = pos_tag(self.list_words, corpus=postag_corpus) + self.list_pos: list[tuple[str, str]] = pos_tag(self.list_words, corpus=postag_corpus) for word, pos in self.list_pos: - self.temp = self.find_synonyms(word, pos, postag_corpus) + self.temp: list[str] = self.find_synonyms(word, pos, postag_corpus) if not self.temp: self.list_synonym.append([word]) else: diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index 2309157bb..3cd874641 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -236,7 +236,7 @@ def _binary_representation(txt: str, verbose: bool = False) -> np.ndarray: return bin_rept -def _find_word_boundaries(bin_reps) -> list[tuple[int, int]]: +def _find_word_boundaries(bin_reps: np.ndarray) -> list[tuple[int, int]]: """Find the starting and ending location of each word. :param str bin_reps: binary representation of a text diff --git a/pythainlp/chat/__init__.py b/pythainlp/chat/__init__.py index 10c25c49d..19c250616 100644 --- a/pythainlp/chat/__init__.py +++ b/pythainlp/chat/__init__.py @@ -3,6 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 """pythainlp.chat""" -__all__ = ["ChatBotModel"] +__all__: list[str] = ["ChatBotModel"] from pythainlp.chat.core import ChatBotModel diff --git a/pythainlp/chat/core.py b/pythainlp/chat/core.py index 4795a43fd..4b4d24380 100644 --- a/pythainlp/chat/core.py +++ b/pythainlp/chat/core.py @@ -7,11 +7,11 @@ class ChatBotModel: - def __init__(self): + def __init__(self) -> None: """Chat using AI generation""" - self.history = [] + self.history: list[tuple[str, str]] = [] - def reset_chat(self): + def reset_chat(self) -> None: """Reset chat by cleaning history""" self.history = [] @@ -21,10 +21,10 @@ def load_model( return_dict: bool = True, load_in_8bit: bool = False, device: str = "cuda", - torch_dtype=torch.float16, + torch_dtype: torch.dtype = torch.float16, offload_folder: str = "./", low_cpu_mem_usage: bool = True, - ): + ) -> None: """Load model :param str model_name: Model name (Now, we support wangchanglm only) diff --git a/pythainlp/classify/__init__.py b/pythainlp/classify/__init__.py index bdb012334..23e3ffba0 100644 --- a/pythainlp/classify/__init__.py +++ b/pythainlp/classify/__init__.py @@ -3,6 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 """pythainlp.classify""" -__all__ = ["GzipModel"] +__all__: list[str] = ["GzipModel"] from pythainlp.classify.param_free import GzipModel diff --git a/pythainlp/classify/param_free.py b/pythainlp/classify/param_free.py index 2bed7fbf3..c9cf1b91d 100644 --- a/pythainlp/classify/param_free.py +++ b/pythainlp/classify/param_free.py @@ -5,9 +5,10 @@ import gzip import json -from typing import Optional +from typing import Any, Optional import numpy as np +from numpy.typing import NDArray class GzipModel: @@ -21,18 +22,21 @@ class GzipModel: Default is empty string. """ + + cx2_list: list[int] + training_data: "NDArray[Any]" def __init__( self, training_data: Optional[list[tuple[str, str]]] = None, model_path: str = "", - ): + ) -> None: if model_path: self.load(model_path) else: self.training_data = np.array(training_data) self.cx2_list = self.train() - def train(self): + def train(self) -> list[int]: temp_list = [] for i in range(len(self.training_data)): temp_list.append( @@ -86,7 +90,7 @@ def predict(self, x1: str, k: int = 1) -> str: return predict_class - def save(self, path: str): + def save(self, path: str) -> None: """:param str path: path to save model""" with open(path, "w", encoding="utf-8") as f: json.dump( @@ -98,7 +102,7 @@ def save(self, path: str): ensure_ascii=False, ) - def load(self, path: str): + def load(self, path: str) -> None: """:param str path: path to load model""" with open(path, "r", encoding="utf-8") as f: data = json.load(f) diff --git a/pythainlp/cli/tag.py b/pythainlp/cli/tag.py index fe98406d9..538555d3f 100644 --- a/pythainlp/cli/tag.py +++ b/pythainlp/cli/tag.py @@ -19,6 +19,7 @@ class SubAppBase: separator: str run: Callable[[list[str]], list[tuple[str, str]]] + args: argparse.Namespace def __init__(self, name: str, argv: Sequence[str]) -> None: parser = argparse.ArgumentParser(**cli.make_usage("tag " + name)) # type: ignore[arg-type] @@ -47,6 +48,9 @@ def __init__(self, name: str, argv: Sequence[str]) -> None: class POSTaggingApp(SubAppBase): + separator: str + run: Callable[[list[str]], list[tuple[str, str]]] + def __init__(self, *args: str, **kwargs: str) -> None: self.separator = "|" self.run = pos_tag diff --git a/pythainlp/cli/tokenize.py b/pythainlp/cli/tokenize.py index 653b77361..efbc9cbea 100644 --- a/pythainlp/cli/tokenize.py +++ b/pythainlp/cli/tokenize.py @@ -32,6 +32,8 @@ class SubAppBase: separator: str algorithm: str run: Callable[..., Any] + keep_whitespace: bool + args: argparse.Namespace def __init__(self, name: str, argv: Sequence[str]) -> None: parser = argparse.ArgumentParser(**cli.make_usage("tokenize " + name)) # type: ignore[arg-type] @@ -85,28 +87,28 @@ def __init__(self, name: str, argv: Sequence[str]) -> None: class WordTokenizationApp(SubAppBase): def __init__(self, *args: str, **kwargs: str) -> None: - self.keep_whitespace = True - self.algorithm = DEFAULT_WORD_TOKENIZE_ENGINE - self.separator = DEFAULT_WORD_TOKEN_SEPARATOR - self.run = word_tokenize + self.keep_whitespace: bool = True + self.algorithm: str = DEFAULT_WORD_TOKENIZE_ENGINE + self.separator: str = DEFAULT_WORD_TOKEN_SEPARATOR + self.run: Callable[..., Any] = word_tokenize super().__init__(*args, **kwargs) class SentenceTokenizationApp(SubAppBase): def __init__(self, *args: str, **kwargs: str) -> None: - self.keep_whitespace = True - self.algorithm = DEFAULT_SENT_TOKENIZE_ENGINE - self.separator = DEFAULT_SENT_TOKEN_SEPARATOR - self.run = sent_tokenize + self.keep_whitespace: bool = True + self.algorithm: str = DEFAULT_SENT_TOKENIZE_ENGINE + self.separator: str = DEFAULT_SENT_TOKEN_SEPARATOR + self.run: Callable[..., Any] = sent_tokenize super().__init__(*args, **kwargs) class SubwordTokenizationApp(SubAppBase): def __init__(self, *args: str, **kwargs: str) -> None: - self.keep_whitespace = True - self.algorithm = DEFAULT_SUBWORD_TOKENIZE_ENGINE - self.separator = DEFAULT_SUBWORD_TOKEN_SEPARATOR - self.run = subword_tokenize + self.keep_whitespace: bool = True + self.algorithm: str = DEFAULT_SUBWORD_TOKENIZE_ENGINE + self.separator: str = DEFAULT_SUBWORD_TOKEN_SEPARATOR + self.run: Callable[..., Any] = subword_tokenize super().__init__(*args, **kwargs) diff --git a/pythainlp/coref/_fastcoref.py b/pythainlp/coref/_fastcoref.py index 2d203aa0a..b43f2f019 100644 --- a/pythainlp/coref/_fastcoref.py +++ b/pythainlp/coref/_fastcoref.py @@ -3,12 +3,18 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from typing import TYPE_CHECKING, Any, Optional + +if TYPE_CHECKING: + from fastcoref.modeling import CorefModel + from spacy.language import Language + class FastCoref: def __init__( self, - model_name, - nlp=None, + model_name: str, + nlp: Optional[Language] = None, device: str = "cpu", type: str = "FCoref", ) -> None: @@ -22,11 +28,11 @@ def __init__( nlp = spacy.blank("th") - self.model_name = model_name - self.nlp = nlp - self.model = _model(self.model_name, device=device, nlp=self.nlp) + self.model_name: str = model_name + self.nlp: Language = nlp + self.model: CorefModel = _model(self.model_name, device=device, nlp=self.nlp) - def _to_json(self, _predict): + def _to_json(self, _predict: Any) -> dict[str, Any]: return { "text": _predict.text, "clusters_string": _predict.get_clusters(as_strings=True), diff --git a/pythainlp/coref/han_coref.py b/pythainlp/coref/han_coref.py index 046c34273..bd7c2d743 100644 --- a/pythainlp/coref/han_coref.py +++ b/pythainlp/coref/han_coref.py @@ -3,11 +3,16 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from spacy.language import Language + from pythainlp.coref._fastcoref import FastCoref class HanCoref(FastCoref): - def __init__(self, device: str = "cpu", nlp=None) -> None: + def __init__(self, device: str = "cpu", nlp: Optional[Language] = None) -> None: super().__init__( model_name="pythainlp/han-coref-v1.0", device=device, nlp=nlp ) diff --git a/pythainlp/corpus/__init__.py b/pythainlp/corpus/__init__.py index 9b9036854..05f09b1fc 100644 --- a/pythainlp/corpus/__init__.py +++ b/pythainlp/corpus/__init__.py @@ -9,7 +9,7 @@ from __future__ import annotations -__all__ = [ +__all__: list[str] = [ "corpus_db_path", "corpus_db_url", "corpus_path", @@ -52,18 +52,18 @@ # Remote and local corpus databases -_CORPUS_DIRNAME = "corpus" -_CORPUS_PATH = os.path.join(get_pythainlp_path(), _CORPUS_DIRNAME) -_CHECK_MODE = os.getenv("PYTHAINLP_READ_MODE") +_CORPUS_DIRNAME: str = "corpus" +_CORPUS_PATH: str = os.path.join(get_pythainlp_path(), _CORPUS_DIRNAME) +_CHECK_MODE: str | None = os.getenv("PYTHAINLP_READ_MODE") # URL of remote corpus catalog -_CORPUS_DB_URL = "https://pythainlp.org/pythainlp-corpus/db.json" +_CORPUS_DB_URL: str = "https://pythainlp.org/pythainlp-corpus/db.json" # filename of local corpus catalog -_CORPUS_DB_FILENAME = "db.json" +_CORPUS_DB_FILENAME: str = "db.json" # full path of local corpus catalog -_CORPUS_DB_PATH = get_full_data_path(_CORPUS_DB_FILENAME) +_CORPUS_DB_PATH: str = get_full_data_path(_CORPUS_DB_FILENAME) # create a local corpus database if it does not already exist if not os.path.exists(_CORPUS_DB_PATH) and _CHECK_MODE != "1": diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 8c952ac57..ad6bf539b 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -33,6 +33,10 @@ class _ResponseWrapper: """Wrapper to provide requests.Response-like interface for urllib response.""" + status_code: int + headers: "http.client.HTTPMessage" + _content: bytes + def __init__(self, response: HTTPResponse) -> None: self.status_code = response.status self.headers = response.headers diff --git a/pythainlp/corpus/util.py b/pythainlp/corpus/util.py index d11e1f7dc..1555b2e14 100644 --- a/pythainlp/corpus/util.py +++ b/pythainlp/corpus/util.py @@ -133,7 +133,7 @@ def revise_newmm_default_wordset( orig_words = thai_words() trie = Trie(orig_words) - def tokenize(text): + def tokenize(text: str) -> list[str]: return newmm.segment(text, custom_dict=trie) revised_words = revise_wordset(tokenize, orig_words, training_data) diff --git a/pythainlp/corpus/wordnet.py b/pythainlp/corpus/wordnet.py index 7f5eb6e78..b0a8cec3b 100644 --- a/pythainlp/corpus/wordnet.py +++ b/pythainlp/corpus/wordnet.py @@ -13,7 +13,7 @@ from __future__ import annotations from collections.abc import Iterable -from typing import Optional +from typing import IO, Optional, Union import nltk @@ -434,7 +434,7 @@ def morphy(form: str, pos: Optional[str] = None) -> str: return wordnet.morphy(form, pos=None) # type: ignore[no-any-return] -def custom_lemmas(tab_file, lang: str) -> None: +def custom_lemmas(tab_file: Union[str, IO[str]], lang: str) -> None: """This function reads a custom tab file (see: http://compling.hss.ntu.edu.sg/omw/) containing mappings of lemmas in the given language. diff --git a/pythainlp/el/_multiel.py b/pythainlp/el/_multiel.py index 2c73b64b0..135d4c763 100644 --- a/pythainlp/el/_multiel.py +++ b/pythainlp/el/_multiel.py @@ -1,15 +1,25 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Union + +if TYPE_CHECKING: + from multiel import BELA class MultiEL: - def __init__(self, model_name="bela", device="cuda"): + model_name: str + device: str + _bela_run: BELA + + def __init__(self, model_name: str = "bela", device: str = "cuda") -> None: self.model_name = model_name self.device = device self.load_model() - def load_model(self): + def load_model(self) -> None: try: from multiel import BELA except ImportError as exc: @@ -18,7 +28,9 @@ def load_model(self): ) from exc self._bela_run = BELA(device=self.device) - def process_batch(self, list_text): + def process_batch( + self, list_text: Union[list[str], str] + ) -> Union[list[dict], str]: if isinstance(list_text, str): list_text = [list_text] - return self._bela_run.process_batch(list_text) + return self._bela_run.process_batch(list_text) # type: ignore[no-any-return] diff --git a/pythainlp/el/core.py b/pythainlp/el/core.py index e2c254147..c393c2773 100644 --- a/pythainlp/el/core.py +++ b/pythainlp/el/core.py @@ -12,7 +12,7 @@ def __init__( model_name: str = "bela", device: str = "cuda", tag: str = "wikidata", - ): + ) -> None: """EntityLinker :param str model_name: model name (bela) @@ -22,9 +22,9 @@ def __init__( You can read about bela model at `https://github.com/PyThaiNLP/MultiEL \ `_. """ - self.model_name = model_name - self.device = device - self.tag = tag + self.model_name: str = model_name + self.device: str = device + self.tag: str = tag if self.model_name not in ["bela"]: raise NotImplementedError( f"EntityLinker doesn't support {model_name} model." @@ -35,7 +35,7 @@ def __init__( ) from pythainlp.el._multiel import MultiEL - self.model = MultiEL(model_name=self.model_name, device=self.device) + self.model: MultiEL = MultiEL(model_name=self.model_name, device=self.device) def get_el( self, list_text: Union[list[str], str] @@ -59,4 +59,4 @@ def get_el( # 'md_scores': [0.30301809310913086, 0.6399497389793396], # 'el_scores': [0.7142490744590759, 0.8657019734382629]}] """ - return self.model.process_batch(list_text) # type: ignore[no-any-return] + return self.model.process_batch(list_text) diff --git a/pythainlp/generate/core.py b/pythainlp/generate/core.py index 86e2cdb21..a789fc70d 100644 --- a/pythainlp/generate/core.py +++ b/pythainlp/generate/core.py @@ -30,19 +30,25 @@ class Unigram: * *oscar* - OSCAR Corpus """ - def __init__(self, name: str = "tnc"): + counts: dict[str, int] + word: list[str] + n: int + prob: dict[str, float] + _word_prob: dict[str, float] + + def __init__(self, name: str = "tnc") -> None: if name == "tnc": - self.counts = tnc_word_freqs_unigram() + self.counts: dict[str, int] = tnc_word_freqs_unigram() elif name == "ttc": self.counts = ttc_word_freqs_unigram() elif name == "oscar": self.counts = oscar_word_freqs_unigram() - self.word = list(self.counts.keys()) - self.n = 0 + self.word: list[str] = list(self.counts.keys()) + self.n: int = 0 for i in self.word: self.n += self.counts[i] - self.prob = {i: self.counts[i] / self.n for i in self.word} - self._word_prob: dict = {} + self.prob: dict[str, float] = {i: self.counts[i] / self.n for i in self.word} + self._word_prob: dict[str, float] = {} def gen_sentence( self, @@ -116,13 +122,19 @@ class Bigram: * *tnc* - Thai National Corpus (default) """ - def __init__(self, name: str = "tnc"): + uni: dict[str, int] + bi: dict[tuple[str, str], int] + uni_keys: list[str] + bi_keys: list[tuple[str, str]] + words: list[str] + + def __init__(self, name: str = "tnc") -> None: if name == "tnc": - self.uni = tnc_word_freqs_unigram() - self.bi = tnc_word_freqs_bigram() - self.uni_keys = list(self.uni.keys()) - self.bi_keys = list(self.bi.keys()) - self.words = [i[-1] for i in self.bi_keys] + self.uni: dict[str, int] = tnc_word_freqs_unigram() + self.bi: dict[tuple[str, str], int] = tnc_word_freqs_bigram() + self.uni_keys: list[str] = list(self.uni.keys()) + self.bi_keys: list[tuple[str, str]] = list(self.bi.keys()) + self.words: list[str] = [i[-1] for i in self.bi_keys] def prob(self, t1: str, t2: str) -> float: """Probability of word @@ -203,15 +215,23 @@ class Trigram: * *tnc* - Thai National Corpus (default) """ - def __init__(self, name: str = "tnc"): + uni: dict[str, int] + bi: dict[tuple[str, str], int] + ti: dict[tuple[str, str, str], int] + uni_keys: list[str] + bi_keys: list[tuple[str, str]] + ti_keys: list[tuple[str, str, str]] + words: list[str] + + def __init__(self, name: str = "tnc") -> None: if name == "tnc": - self.uni = tnc_word_freqs_unigram() - self.bi = tnc_word_freqs_bigram() - self.ti = tnc_word_freqs_trigram() - self.uni_keys = list(self.uni.keys()) - self.bi_keys = list(self.bi.keys()) - self.ti_keys = list(self.ti.keys()) - self.words = [i[-1] for i in self.bi_keys] + self.uni: dict[str, int] = tnc_word_freqs_unigram() + self.bi: dict[tuple[str, str], int] = tnc_word_freqs_bigram() + self.ti: dict[tuple[str, str, str], int] = tnc_word_freqs_trigram() + self.uni_keys: list[str] = list(self.uni.keys()) + self.bi_keys: list[tuple[str, str]] = list(self.bi.keys()) + self.ti_keys: list[tuple[str, str, str]] = list(self.ti.keys()) + self.words: list[str] = [i[-1] for i in self.bi_keys] def prob(self, t1: str, t2: str, t3: str) -> float: """Probability of word diff --git a/pythainlp/generate/wangchanglm.py b/pythainlp/generate/wangchanglm.py index 51ef44261..7f8df7d0f 100644 --- a/pythainlp/generate/wangchanglm.py +++ b/pythainlp/generate/wangchanglm.py @@ -4,15 +4,31 @@ from __future__ import annotations import re +from typing import TYPE_CHECKING import torch +if TYPE_CHECKING: + import pandas as pd + from transformers import PreTrainedModel, PreTrainedTokenizerBase + class WangChanGLM: - def __init__(self): - self.exclude_pattern = re.compile(r"[^ก-๙]+") - self.stop_token = "\n" # noqa: S105 - self.PROMPT_DICT = { + exclude_pattern: "re.Pattern" + stop_token: str + PROMPT_DICT: dict[str, str] + device: str + torch_dtype: "torch.dtype" + model_path: str + model: "PreTrainedModel" + tokenizer: "PreTrainedTokenizerBase" + df: "pd.DataFrame" + exclude_ids: list[int] + + def __init__(self) -> None: + self.exclude_pattern: "re.Pattern" = re.compile(r"[^ก-๙]+") + self.stop_token: str = "\n" # noqa: S105 + self.PROMPT_DICT: dict[str, str] = { "prompt_input": ( ": {input}\n: {instruction}\n: " ), @@ -29,10 +45,10 @@ def load_model( return_dict: bool = True, load_in_8bit: bool = False, device: str = "cuda", - torch_dtype=torch.float16, + torch_dtype: torch.dtype = torch.float16, offload_folder: str = "./", low_cpu_mem_usage: bool = True, - ): + ) -> None: """Load model :param str model_path: model path @@ -46,10 +62,10 @@ def load_model( import pandas as pd from transformers import AutoModelForCausalLM, AutoTokenizer - self.device = device - self.torch_dtype = torch_dtype - self.model_path = model_path - self.model = AutoModelForCausalLM.from_pretrained( + self.device: str = device + self.torch_dtype: "torch.dtype" = torch_dtype + self.model_path: str = model_path + self.model: "PreTrainedModel" = AutoModelForCausalLM.from_pretrained( self.model_path, return_dict=return_dict, load_in_8bit=load_in_8bit, @@ -58,12 +74,12 @@ def load_model( offload_folder=offload_folder, low_cpu_mem_usage=low_cpu_mem_usage, ) - self.tokenizer = AutoTokenizer.from_pretrained(self.model_path) - self.df = pd.DataFrame( + self.tokenizer: "PreTrainedTokenizerBase" = AutoTokenizer.from_pretrained(self.model_path) + self.df: "pd.DataFrame" = pd.DataFrame( self.tokenizer.vocab.items(), columns=["text", "idx"] ) self.df["is_exclude"] = self.df.text.map(self.is_exclude) - self.exclude_ids = self.df[self.df.is_exclude is True].idx.tolist() + self.exclude_ids: list[int] = self.df[self.df.is_exclude is True].idx.tolist() def gen_instruct( self, @@ -76,7 +92,7 @@ def gen_instruct( typical_p: float = 1.0, thai_only: bool = True, skip_special_tokens: bool = True, - ): + ) -> str: """Generate Instruct :param str text: text @@ -125,7 +141,7 @@ def instruct_generate( self, instruct: str, context: str = "", - max_new_tokens=512, + max_new_tokens: int = 512, temperature: float = 0.9, top_p: float = 0.95, top_k: int = 50, @@ -133,7 +149,7 @@ def instruct_generate( typical_p: float = 1, thai_only: bool = True, skip_special_tokens: bool = True, - ): + ) -> str: """Generate Instruct :param str instruct: Instruct diff --git a/pythainlp/khavee/__init__.py b/pythainlp/khavee/__init__.py index e4882747b..158125ddc 100644 --- a/pythainlp/khavee/__init__.py +++ b/pythainlp/khavee/__init__.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 - -__all__ = ["KhaveeVerifier"] +__all__: list[str] = ["KhaveeVerifier"] from pythainlp.khavee.core import KhaveeVerifier diff --git a/pythainlp/khavee/core.py b/pythainlp/khavee/core.py index 06903775e..b40c349a2 100644 --- a/pythainlp/khavee/core.py +++ b/pythainlp/khavee/core.py @@ -12,7 +12,7 @@ class KhaveeVerifier: - def __init__(self): + def __init__(self) -> None: """ KhaveeVerifier: Thai Poetry verifier """ @@ -356,7 +356,7 @@ def is_sumpus(self, word1: str, word2: str) -> bool: marttra2 = "กา" return bool(marttra1 == marttra2 and sara1 == sara2) - def check_karu_lahu(self, text): + def check_karu_lahu(self, text: str) -> str: if ( self.check_marttra(text) != "กา" or ( diff --git a/pythainlp/morpheme/__init__.py b/pythainlp/morpheme/__init__.py index 191985189..7508d8350 100644 --- a/pythainlp/morpheme/__init__.py +++ b/pythainlp/morpheme/__init__.py @@ -4,6 +4,6 @@ """PyThaiNLP morpheme""" -__all__ = ["nighit", "is_native_thai"] +__all__: list[str] = ["nighit", "is_native_thai"] from pythainlp.morpheme.thaiwordcheck import is_native_thai from pythainlp.morpheme.word_formation import nighit diff --git a/pythainlp/morpheme/thaiwordcheck.py b/pythainlp/morpheme/thaiwordcheck.py index 6891b1cf0..50f4eb44b 100644 --- a/pythainlp/morpheme/thaiwordcheck.py +++ b/pythainlp/morpheme/thaiwordcheck.py @@ -17,11 +17,12 @@ from __future__ import annotations import re +from typing import Pattern -_THANTHAKHAT_CHAR = "\u0e4c" # Thanthakhat (cancellation of sound) +_THANTHAKHAT_CHAR: str = "\u0e4c" # Thanthakhat (cancellation of sound) # Non-native Thai characters -_TH_NON_NATIVE_CHARS = { +_TH_NON_NATIVE_CHARS: set[str] = { "ฆ", "ณ", "ฌ", @@ -38,10 +39,10 @@ } # Native Thai final consonants -_TH_NATIVE_FINALS = {"ก", "ด", "บ", "น", "ง", "ม", "ย", "ว"} +_TH_NATIVE_FINALS: set[str] = {"ก", "ด", "บ", "น", "ง", "ม", "ย", "ว"} # Known native Thai words (exceptions) -_TH_NATIVE_WORDS = { +_TH_NATIVE_WORDS: set[str] = { "ฆ่า", "เฆี่ยน", "ศึก", diff --git a/pythainlp/parse/__init__.py b/pythainlp/parse/__init__.py index 4aca909a7..c8acdadb2 100644 --- a/pythainlp/parse/__init__.py +++ b/pythainlp/parse/__init__.py @@ -1,9 +1,8 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 - """PyThaiNLP Parse""" -__all__ = ["dependency_parsing"] +__all__: list[str] = ["dependency_parsing"] from pythainlp.parse.core import dependency_parsing diff --git a/pythainlp/parse/esupar_engine.py b/pythainlp/parse/esupar_engine.py index 9dd875f03..65d315b29 100644 --- a/pythainlp/parse/esupar_engine.py +++ b/pythainlp/parse/esupar_engine.py @@ -5,19 +5,22 @@ from __future__ import annotations -from typing import List, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Union try: import esupar except ImportError: raise ImportError("Import Error; Install esupar by pip install esupar") +if TYPE_CHECKING: + from esupar import Model + class Parse: def __init__(self, model: Optional[str] = "th") -> None: if model is None: model = "th" - self.nlp = esupar.load(model) + self.nlp: Model = esupar.load(model) # type: ignore[assignment] def __call__( self, text: str, tag: str = "str" diff --git a/pythainlp/parse/spacy_thai_engine.py b/pythainlp/parse/spacy_thai_engine.py index cffae5cc5..8c28256ce 100644 --- a/pythainlp/parse/spacy_thai_engine.py +++ b/pythainlp/parse/spacy_thai_engine.py @@ -6,14 +6,17 @@ from __future__ import annotations -from typing import List, Union +from typing import TYPE_CHECKING, List, Union import spacy_thai +if TYPE_CHECKING: + from spacy_thai import Language + class Parse: def __init__(self, model: str = "th") -> None: - self.nlp = spacy_thai.load() + self.nlp: Language = spacy_thai.load() # type: ignore[assignment] def __call__( self, text: str, tag: str = "str" diff --git a/pythainlp/parse/transformers_ud.py b/pythainlp/parse/transformers_ud.py index 076de9e85..fc5aac53c 100644 --- a/pythainlp/parse/transformers_ud.py +++ b/pythainlp/parse/transformers_ud.py @@ -12,7 +12,14 @@ from __future__ import annotations import os -from typing import Optional, Union +from typing import TYPE_CHECKING, Optional, Union + +if TYPE_CHECKING: + from transformers import ( + AutoTokenizer, + AutoModelForQuestionAnswering, + TokenClassificationPipeline, + ) class Parse: @@ -30,8 +37,8 @@ def __init__( if model is None: model = "KoichiYasuoka/deberta-base-thai-ud-head" - self.tokenizer = AutoTokenizer.from_pretrained(model) - self.model = AutoModelForQuestionAnswering.from_pretrained(model) + self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(model) + self.model: AutoModelForQuestionAnswering = AutoModelForQuestionAnswering.from_pretrained(model) x = AutoModelForTokenClassification.from_pretrained if os.path.isdir(model): d, t = ( @@ -47,10 +54,10 @@ def __init__( cached_file(model, "tagger/config.json") ) t = x(cached_file(model, "tagger/pytorch_model.bin"), config=s) - self.deprel = TokenClassificationPipeline( + self.deprel: TokenClassificationPipeline = TokenClassificationPipeline( model=d, tokenizer=self.tokenizer, aggregation_strategy="simple" ) - self.tagger = TokenClassificationPipeline( + self.tagger: TokenClassificationPipeline = TokenClassificationPipeline( model=t, tokenizer=self.tokenizer ) diff --git a/pythainlp/parse/ud_goeswith.py b/pythainlp/parse/ud_goeswith.py index 5ca1c2d81..e2fb3c02a 100644 --- a/pythainlp/parse/ud_goeswith.py +++ b/pythainlp/parse/ud_goeswith.py @@ -11,12 +11,15 @@ from __future__ import annotations -from typing import List, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Union import numpy as np import torch from transformers import AutoModelForTokenClassification, AutoTokenizer +if TYPE_CHECKING: + pass + class Parse: def __init__( @@ -25,8 +28,8 @@ def __init__( ) -> None: if model is None: model = "KoichiYasuoka/deberta-base-thai-ud-goeswith" - self.tokenizer = AutoTokenizer.from_pretrained(model) - self.model = AutoModelForTokenClassification.from_pretrained(model) + self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(model) + self.model: AutoModelForTokenClassification = AutoModelForTokenClassification.from_pretrained(model) def __call__( self, text: str, tag: str = "str" diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index f622b2916..8056b75e9 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -22,7 +22,7 @@ class ThaiTextProcessor: - def __init__(self): + def __init__(self) -> None: ( self._TK_UNK, self._TK_REP, @@ -30,7 +30,7 @@ def __init__(self): self._TK_URL, self._TK_END, ) = " ".split() - self.SPACE_SPECIAL_TOKEN = "<_>" # noqa: S105 + self.SPACE_SPECIAL_TOKEN: str = "<_>" # noqa: S105 def replace_url(self, text: str) -> str: """Replace url in `text` with TK_URL (https://stackoverflow.com/a/6041965) @@ -126,7 +126,7 @@ def replace_rep_after(self, text: str) -> str: 'กา' """ - def _replace_rep(m): + def _replace_rep(m: re.Match[str]) -> str: c, cc = m.groups() return f"{c}" @@ -206,16 +206,16 @@ def __init__(self) -> None: pipeline, ) - self.tokenizer = AutoTokenizer.from_pretrained(_model_name) - self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained( + self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(_model_name) # type: ignore[assignment] + self.model_for_masked_lm: AutoModelForMaskedLM = AutoModelForMaskedLM.from_pretrained( _model_name - ) - self.model = pipeline( + ) # type: ignore[assignment] + self.model: any = pipeline( # transformers.Pipeline "fill-mask", tokenizer=self.tokenizer, model=self.model_for_masked_lm, ) - self.processor = ThaiTextProcessor() + self.processor: ThaiTextProcessor = ThaiTextProcessor() def generate( self, @@ -303,8 +303,8 @@ def __init__(self, model: str = "lunarlist/pos_thai_phayathai") -> None: AutoTokenizer, ) - self.tokenizer = AutoTokenizer.from_pretrained(model) - self.model = AutoModelForTokenClassification.from_pretrained(model) + self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(model) # type: ignore[assignment] + self.model: AutoModelForTokenClassification = AutoModelForTokenClassification.from_pretrained(model) # type: ignore[assignment] def get_tag( self, sentence: str, strategy: str = "simple" @@ -346,8 +346,8 @@ def __init__(self, model: str = "Pavarissy/phayathaibert-thainer") -> None: AutoTokenizer, ) - self.tokenizer = AutoTokenizer.from_pretrained(model) - self.model = AutoModelForTokenClassification.from_pretrained(model) + self.tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(model) # type: ignore[assignment] + self.model: AutoModelForTokenClassification = AutoModelForTokenClassification.from_pretrained(model) # type: ignore[assignment] def get_ner( self, diff --git a/pythainlp/soundex/__init__.py b/pythainlp/soundex/__init__.py index 8fee8d18e..94e86c3db 100644 --- a/pythainlp/soundex/__init__.py +++ b/pythainlp/soundex/__init__.py @@ -6,7 +6,7 @@ Has three systems to choose from: Udom83 (default), LK82, and MetaSound """ -__all__ = [ +__all__: list[str] = [ "complete_soundex", "complete_soundex_similarity", "lk82", @@ -25,6 +25,6 @@ from pythainlp.soundex.prayut_and_somchaip import prayut_and_somchaip from pythainlp.soundex.udom83 import udom83 -DEFAULT_SOUNDEX_ENGINE = "udom83" +DEFAULT_SOUNDEX_ENGINE: str = "udom83" from pythainlp.soundex.core import soundex diff --git a/pythainlp/soundex/complete_soundex.py b/pythainlp/soundex/complete_soundex.py index 3dc63e7e2..7b5a28464 100644 --- a/pythainlp/soundex/complete_soundex.py +++ b/pythainlp/soundex/complete_soundex.py @@ -43,12 +43,12 @@ class CompleteSoundex: by Chalermpol Tapsai, Phayung Meesad, and Choochart Haruechaiyasak (2020). """ - def __init__(self): + def __init__(self) -> None: # Thai consonants for pattern matching - self.thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬฮอ" + self.thai_consonants: str = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬฮอ" # 1. Maps (Tables 5.1 - 5.4) - self.initial_map = { + self.initial_map: dict[str, str] = { "ก": "กก", "ข": "คข", "ฃ": "คข", @@ -96,7 +96,7 @@ def __init__(self): "อ": "ออ", } - self.vowel_map = { + self.vowel_map: dict[str, str] = { "ะ": "1A", "ั": "1A", "รร": "1A", @@ -132,7 +132,7 @@ def __init__(self): "ว": "CX", } - self.final_map = { + self.final_map: dict[str, str] = { "ก": "ก", "ข": "ก", "ค": "ก", @@ -170,7 +170,7 @@ def __init__(self): "ว": "ว", } - self.tone_map = {"่": "1", "้": "2", "๊": "3", "๋": "4"} + self.tone_map: dict[str, str] = {"่": "1", "้": "2", "๊": "3", "๋": "4"} def clean_text(self, text: str) -> str: """Remove silent characters (karan/thanthakhat) from text.""" diff --git a/pythainlp/soundex/lk82.py b/pythainlp/soundex/lk82.py index 8d4d9561c..0b0b8c642 100644 --- a/pythainlp/soundex/lk82.py +++ b/pythainlp/soundex/lk82.py @@ -17,24 +17,25 @@ from __future__ import annotations import re +from typing import Pattern from pythainlp.util import remove_tonemark -_TRANS1 = str.maketrans( +_TRANS1: dict[int, int] = str.maketrans( "กขฃคฅฆงจฉชฌซศษสญยฎดฏตณนฐฑฒถทธบปผพภฝฟมรลฬฤฦวหฮอ", "กกกกกกงจชชชซซซซยยดดตตนนททททททบปพพพฟฟมรรรรรวหหอ", ) -_TRANS2 = str.maketrans( +_TRANS2: dict[int, int] = str.maketrans( "กขฃคฅฆงจฉชซฌฎฏฐฑฒดตถทธศษสญณนรลฬฤฦบปพฟภผฝมำยวไใหฮาๅึืเแโุูอ", "1111112333333333333333333444444445555555667777889AAABCDEEF", ) # silenced -_RE_KARANT = re.compile(r"จน์|มณ์|ณฑ์|ทร์|ตร์|[ก-ฮ]์|[ก-ฮ][ะ-ู]์") +_RE_KARANT: Pattern[str] = re.compile(r"จน์|มณ์|ณฑ์|ทร์|ตร์|[ก-ฮ]์|[ก-ฮ][ะ-ู]์") # signs, symbols, vowel that has no explicit sounds # Paiyannoi, Phinthu, Maiyamok, Maitaikhu, Nikhahit -_RE_SIGN = re.compile(r"[\u0e2f\u0e3a\u0e46\u0e47\u0e4d]") +_RE_SIGN: Pattern[str] = re.compile(r"[\u0e2f\u0e3a\u0e46\u0e47\u0e4d]") def lk82(text: str) -> str: diff --git a/pythainlp/soundex/metasound.py b/pythainlp/soundex/metasound.py index e5876ef27..590c470bb 100644 --- a/pythainlp/soundex/metasound.py +++ b/pythainlp/soundex/metasound.py @@ -13,16 +13,16 @@ from __future__ import annotations -_CONS_THANTHAKHAT = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ์" -_THANTHAKHAT = "์" # \u0e4c -_C1 = "กขฃคฆฅ" # sound K -> coded letter 1 -_C2 = "จฉชฌซฐทฒดฎตสศษ" # D -> 2 -_C3 = "ฟฝพผภบป" # B -> 3 -_C4 = "ง" # NG -> 4 -_C5 = "ลฬรนณฦญ" # N -> 5 -_C6 = "ม" # M -> 6 -_C7 = "ย" # Y -> 7 -_C8 = "ว" # W -> 8 +_CONS_THANTHAKHAT: str = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ์" +_THANTHAKHAT: str = "์" # \u0e4c +_C1: str = "กขฃคฆฅ" # sound K -> coded letter 1 +_C2: str = "จฉชฌซฐทฒดฎตสศษ" # D -> 2 +_C3: str = "ฟฝพผภบป" # B -> 3 +_C4: str = "ง" # NG -> 4 +_C5: str = "ลฬรนณฦญ" # N -> 5 +_C6: str = "ม" # M -> 6 +_C7: str = "ย" # Y -> 7 +_C8: str = "ว" # W -> 8 def metasound(text: str, length: int = 4) -> str: diff --git a/pythainlp/soundex/prayut_and_somchaip.py b/pythainlp/soundex/prayut_and_somchaip.py index c71507043..f26aec432 100644 --- a/pythainlp/soundex/prayut_and_somchaip.py +++ b/pythainlp/soundex/prayut_and_somchaip.py @@ -16,18 +16,18 @@ from pythainlp import thai_characters -_C0 = "AEIOUHWYอ" -_C1 = "BFPVบฝฟปผพภว" -_C2 = "CGJKQSXZขฃคฅฆฉขฌกจซศษส" -_C3 = "DTฎดฏตฐฑฒถทธ" -_C4 = "Lลฬ" -_C5 = "MNมณน" -_C6 = "Rร" -_C7 = "AEIOUอ" -_C8 = "Hหฮ" -_C1_1 = "Wว" -_C9 = "Yยญ" -_C52 = "ง" +_C0: str = "AEIOUHWYอ" +_C1: str = "BFPVบฝฟปผพภว" +_C2: str = "CGJKQSXZขฃคฅฆฉขฌกจซศษส" +_C3: str = "DTฎดฏตฐฑฒถทธ" +_C4: str = "Lลฬ" +_C5: str = "MNมณน" +_C6: str = "Rร" +_C7: str = "AEIOUอ" +_C8: str = "Hหฮ" +_C1_1: str = "Wว" +_C9: str = "Yยญ" +_C52: str = "ง" def prayut_and_somchaip(text: str, length: int = 4) -> str: diff --git a/pythainlp/soundex/udom83.py b/pythainlp/soundex/udom83.py index e5736c6a5..201a66830 100644 --- a/pythainlp/soundex/udom83.py +++ b/pythainlp/soundex/udom83.py @@ -18,31 +18,32 @@ from __future__ import annotations import re +from typing import Pattern from pythainlp import thai_consonants -_THANTHAKHAT = "\u0e4c" -_RE_1 = re.compile(r"รร([\u0e40-\u0e44])") # เ-ไ -_RE_2 = re.compile(f"รร([{thai_consonants}][{thai_consonants}\u0e40-\u0e44])") -_RE_3 = re.compile(f"รร([{thai_consonants}][\u0e30-\u0e39\u0e48-\u0e4c])") -_RE_4 = re.compile(r"รร") -_RE_5 = re.compile(f"ไ([{thai_consonants}]ย)") -_RE_6 = re.compile(f"[ไใ]([{thai_consonants}])") -_RE_7 = re.compile(r"\u0e33(ม[\u0e30-\u0e39])") -_RE_8 = re.compile(r"\u0e33ม") -_RE_9 = re.compile(r"\u0e33") # ำ -_RE_10 = re.compile( +_THANTHAKHAT: str = "\u0e4c" +_RE_1: Pattern[str] = re.compile(r"รร([\u0e40-\u0e44])") # เ-ไ +_RE_2: Pattern[str] = re.compile(f"รร([{thai_consonants}][{thai_consonants}\u0e40-\u0e44])") +_RE_3: Pattern[str] = re.compile(f"รร([{thai_consonants}][\u0e30-\u0e39\u0e48-\u0e4c])") +_RE_4: Pattern[str] = re.compile(r"รร") +_RE_5: Pattern[str] = re.compile(f"ไ([{thai_consonants}]ย)") +_RE_6: Pattern[str] = re.compile(f"[ไใ]([{thai_consonants}])") +_RE_7: Pattern[str] = re.compile(r"\u0e33(ม[\u0e30-\u0e39])") +_RE_8: Pattern[str] = re.compile(r"\u0e33ม") +_RE_9: Pattern[str] = re.compile(r"\u0e33") # ำ +_RE_10: Pattern[str] = re.compile( f"จน์|มณ์|ณฑ์|ทร์|ตร์|" f"[{thai_consonants}]{_THANTHAKHAT}|[{thai_consonants}]" f"[\u0e30-\u0e39]{_THANTHAKHAT}" ) -_RE_11 = re.compile(r"[\u0e30-\u0e4c]") +_RE_11: Pattern[str] = re.compile(r"[\u0e30-\u0e4c]") -_TRANS1 = str.maketrans( +_TRANS1: dict[int, int] = str.maketrans( "กขฃคฅฆงจฉชฌซศษสฎดฏตฐฑฒถทธณนบปผพภฝฟมญยรลฬฤฦวอหฮ", "กขขขขขงจชชชสสสสดดตตททททททนนบปพพพฟฟมยยรรรรรวอฮฮ", ) -_TRANS2 = str.maketrans( +_TRANS2: dict[int, int] = str.maketrans( "มวำกขฃคฅฆงยญณนฎฏดตศษสบปพภผฝฟหอฮจฉชซฌฐฑฒถทธรฤลฦ", "0001111112233344444445555666666777778888889999", ) diff --git a/pythainlp/spell/__init__.py b/pythainlp/spell/__init__.py index 36ab4b132..6c78dfa9c 100644 --- a/pythainlp/spell/__init__.py +++ b/pythainlp/spell/__init__.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 """Spell checking and correction.""" -__all__ = [ +__all__: list[str] = [ "DEFAULT_SPELL_CHECKER", "NorvigSpellChecker", "correct", @@ -14,8 +14,9 @@ ] from pythainlp.spell.pn import NorvigSpellChecker +from typing import Type -DEFAULT_SPELL_CHECKER = NorvigSpellChecker +DEFAULT_SPELL_CHECKER: Type[NorvigSpellChecker] = NorvigSpellChecker # these imports are placed here to avoid circular imports from pythainlp.spell.core import correct, correct_sent, spell, spell_sent diff --git a/pythainlp/spell/pn.py b/pythainlp/spell/pn.py index c7db683e2..ab9a8e6ea 100644 --- a/pythainlp/spell/pn.py +++ b/pythainlp/spell/pn.py @@ -129,7 +129,7 @@ def __init__( min_len: int = 2, max_len: int = 40, dict_filter: Optional[Callable[[str], bool]] = _is_thai_and_not_num, - ): + ) -> None: """Initializes Peter Norvig's spell checker object. Spelling dictionary can be customized. By default, spelling dictionary is from @@ -173,9 +173,9 @@ def __init__( custom_dict, min_freq, min_len, max_len, dict_filter ) - self.__WORDS = Counter(dict(custom_dict)) + self.__WORDS: Counter[str] = Counter(dict(custom_dict)) self.__WORDS += Counter() # remove zero and negative counts - self.__WORDS_TOTAL = sum(self.__WORDS.values()) + self.__WORDS_TOTAL: int = sum(self.__WORDS.values()) def dictionary(self) -> ItemsView[str, int]: """Returns the spelling dictionary currently used by this spell checker diff --git a/pythainlp/spell/symspellpy.py b/pythainlp/spell/symspellpy.py index 7064fbf60..1904ed304 100644 --- a/pythainlp/spell/symspellpy.py +++ b/pythainlp/spell/symspellpy.py @@ -108,7 +108,9 @@ def spell_sent( return list_new -def correct_sent(list_words: list[str], max_edit_distance=1) -> list[str]: +def correct_sent( + list_words: list[str], max_edit_distance: int = 1 +) -> list[str]: return [ i[0] for i in spell_sent(list_words, max_edit_distance=max_edit_distance) diff --git a/pythainlp/spell/wanchanberta_thai_grammarly.py b/pythainlp/spell/wanchanberta_thai_grammarly.py index 83f0e50fe..1f3e5fc6d 100644 --- a/pythainlp/spell/wanchanberta_thai_grammarly.py +++ b/pythainlp/spell/wanchanberta_thai_grammarly.py @@ -12,7 +12,7 @@ from __future__ import annotations -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional import torch from transformers import ( @@ -21,6 +21,9 @@ BertForTokenClassification, ) +if TYPE_CHECKING: + pass + use_cuda = torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") tokenizer = AutoTokenizer.from_pretrained( @@ -31,7 +34,7 @@ class BertModel(torch.nn.Module): def __init__(self) -> None: super().__init__() - self.bert = BertForTokenClassification.from_pretrained( + self.bert: BertForTokenClassification = BertForTokenClassification.from_pretrained( "bookpanda/wangchanberta-base-att-spm-uncased-tagging" ) diff --git a/pythainlp/spell/words_spelling_correction.py b/pythainlp/spell/words_spelling_correction.py index d2f55d3bb..6180b2718 100644 --- a/pythainlp/spell/words_spelling_correction.py +++ b/pythainlp/spell/words_spelling_correction.py @@ -4,7 +4,12 @@ from __future__ import annotations import os -from typing import Any, Union +from typing import TYPE_CHECKING, Union + +if TYPE_CHECKING: + import numpy as np + from numpy.typing import NDArray + from onnxruntime import InferenceSession from pythainlp.corpus import get_hf_hub @@ -15,6 +20,18 @@ class FastTextEncoder: model for nearest neighbor suggestions. """ + model_dir: str + nn_model_path: str + bucket: int + nb_words: int + minn: int + maxn: int + vocabulary: list[str] + embeddings: NDArray[np.float32] + words_for_suggestion: NDArray[np.str_] + nn_session: InferenceSession + embedding_dim: int + # --- Initialization and Data Loading --- def __init__( @@ -42,8 +59,6 @@ def __init__( """ try: import numpy as np - - self.np = np except ModuleNotFoundError: raise ModuleNotFoundError(""" Please installing the package via 'pip install numpy onnxruntime'. @@ -61,11 +76,13 @@ def __init__( self.vocabulary, self.embeddings = self._load_embeddings() self.words_for_suggestion = self._load_suggestion_words(words_list) self.nn_session = self._load_onnx_session(nn_model_path) - self.embedding_dim = self.embeddings.shape[1] + self.embedding_dim: int = self.embeddings.shape[1] - def _load_embeddings(self) -> tuple[list[str], Any]: + def _load_embeddings(self) -> tuple[list[str], NDArray[np.float32]]: """Loads embeddings matrix and vocabulary list.""" - input_matrix = self.np.load( + import numpy as np + + input_matrix = np.load( os.path.join(self.model_dir, "embeddings.npy") ) words = [] @@ -75,12 +92,14 @@ def _load_embeddings(self) -> tuple[list[str], Any]: words.append(line.rstrip()) return words, input_matrix - def _load_suggestion_words(self, words_list: list[str]) -> Any: + def _load_suggestion_words(self, words_list: list[str]) -> NDArray[np.str_]: """Loads the list of words used for suggestions.""" - words = self.np.array(words_list) + import numpy as np + + words = np.array(words_list) return words - def _load_onnx_session(self, onnx_path: str) -> Any: + def _load_onnx_session(self, onnx_path: str) -> InferenceSession: """Loads the ONNX inference session.""" # Note: Using providers=["CPUExecutionProvider"] for platform independence import onnxruntime as rt @@ -101,7 +120,7 @@ def _get_hash(self, subword: str) -> int: h = (h * 16777619) % 2**32 # FNV-1a prime return h % self.bucket + self.nb_words - def _get_subwords(self, word: str) -> tuple[list[str], Any]: + def _get_subwords(self, word: str) -> tuple[list[str], NDArray[np.int_]]: """Extracts subwords and their corresponding indices for a given word.""" _word = "<" + word + ">" _subwords = [] @@ -112,7 +131,8 @@ def _get_subwords(self, word: str) -> tuple[list[str], Any]: _subwords.append(word) _subword_ids.append(self.vocabulary.index(word)) if word == "": - return _subwords, self.np.array(_subword_ids) + import numpy as np + return _subwords, np.array(_subword_ids) # 2. Extract n-grams (subwords) and get their hash indices for ngram_start in range(0, len(_word)): @@ -126,25 +146,28 @@ def _get_subwords(self, word: str) -> tuple[list[str], Any]: _subwords.append(_candidate_subword) _subword_ids.append(self._get_hash(_candidate_subword)) - return _subwords, self.np.array(_subword_ids) + import numpy as np + return _subwords, np.array(_subword_ids) - def get_word_vector(self, word: str) -> Any: + def get_word_vector(self, word: str) -> NDArray[np.float32]: """Computes the normalized vector for a single word.""" + import numpy as np + # subword_ids[1] contains the array of indices for the word and its subwords subword_ids = self._get_subwords(word)[1] # Check if the array of subword indices is empty if subword_ids.size == 0: # Return a 300-dimensional zero vector if no word/subword is found. - return self.np.zeros(self.embedding_dim) + return np.zeros(self.embedding_dim) # Compute the mean of the embeddings for all subword indices - vector = self.np.mean( + vector = np.mean( [self.embeddings[s] for s in subword_ids], axis=0 ) # Normalize the vector - norm = self.np.linalg.norm(vector) + norm = np.linalg.norm(vector) if norm > 0: vector /= norm @@ -167,8 +190,10 @@ def _tokenize(self, sentence: str) -> list[str]: tokens.append(word) return tokens - def get_sentence_vector(self, line: str) -> Any: + def get_sentence_vector(self, line: str) -> NDArray[np.float32]: """Computes the mean vector for a sentence.""" + import numpy as np + tokens = self._tokenize(line) vectors = [] for t in tokens: @@ -178,9 +203,9 @@ def get_sentence_vector(self, line: str) -> Any: # If the sentence was empty and resulted in no vectors, return a zero vector if not vectors: - return self.np.zeros(self.embedding_dim) + return np.zeros(self.embedding_dim) - return self.np.mean(vectors, axis=0) + return np.mean(vectors, axis=0) # --- Nearest Neighbor Method --- @@ -217,7 +242,8 @@ def get_word_suggestion( ] # Convert to numpy array for ONNX input (ensure float32) - input_data = self.np.array(word_input_vecs, dtype=self.np.float32) + import numpy as np + input_data = np.array(word_input_vecs, dtype=np.float32) # Run ONNX inference indices = self.nn_session.run(None, {"X": input_data})[0] @@ -230,9 +256,9 @@ def get_word_suggestion( class Words_Spelling_Correction(FastTextEncoder): def __init__(self) -> None: - self.model_name = "pythainlp/word-spelling-correction-char2vec" - self.model_path = get_hf_hub(self.model_name) - self.model_onnx = get_hf_hub(self.model_name, "nearest_neighbors.onnx") + self.model_name: str = "pythainlp/word-spelling-correction-char2vec" + self.model_path: str = get_hf_hub(self.model_name) # type: ignore[assignment] + self.model_onnx: str = get_hf_hub(self.model_name, "nearest_neighbors.onnx") # type: ignore[assignment] with open( get_hf_hub( self.model_name, "list_word-spelling-correction-char2vec.txt" diff --git a/pythainlp/summarize/__init__.py b/pythainlp/summarize/__init__.py index 6265d58ed..99805f5ae 100644 --- a/pythainlp/summarize/__init__.py +++ b/pythainlp/summarize/__init__.py @@ -3,14 +3,14 @@ # SPDX-License-Identifier: Apache-2.0 """Text summarization""" -__all__ = [ +__all__: list[str] = [ "extract_keywords", "summarize", ] -DEFAULT_SUMMARIZE_ENGINE = "frequency" -CPE_KMUTT_THAI_SENTENCE_SUM = "mt5-cpe-kmutt-thai-sentence-sum" -DEFAULT_KEYWORD_EXTRACTION_ENGINE = "keybert" +DEFAULT_SUMMARIZE_ENGINE: str = "frequency" +CPE_KMUTT_THAI_SENTENCE_SUM: str = "mt5-cpe-kmutt-thai-sentence-sum" +DEFAULT_KEYWORD_EXTRACTION_ENGINE: str = "keybert" # these imports are placed here to avoid circular imports from pythainlp.summarize.core import extract_keywords, summarize diff --git a/pythainlp/summarize/freq.py b/pythainlp/summarize/freq.py index 5b9485047..1e491b7a4 100644 --- a/pythainlp/summarize/freq.py +++ b/pythainlp/summarize/freq.py @@ -17,14 +17,19 @@ class FrequencySummarizer: - def __init__(self, min_cut: float = 0.1, max_cut: float = 0.9): - self.__min_cut = min_cut - self.__max_cut = max_cut - self.__stopwords = set(punctuation).union(_STOPWORDS) + __min_cut: float + __max_cut: float + __stopwords: set[str] + __freq: "defaultdict[str, float]" + + def __init__(self, min_cut: float = 0.1, max_cut: float = 0.9) -> None: + self.__min_cut: float = min_cut + self.__max_cut: float = max_cut + self.__stopwords: set[str] = set(punctuation).union(_STOPWORDS) @staticmethod - def __rank(ranking, n: int): - return nlargest(n, ranking, key=ranking.get) + def __rank(ranking: dict, n: int) -> list: + return nlargest(n, ranking, key=ranking.get) # type: ignore[arg-type] def __compute_frequencies( self, word_tokenized_sents: list[list[str]] @@ -56,7 +61,7 @@ def summarize( word_tokenized_sents = [ word_tokenize(sent, engine=tokenizer) for sent in sents ] - self.__freq = self.__compute_frequencies(word_tokenized_sents) + self.__freq: "defaultdict[str, float]" = self.__compute_frequencies(word_tokenized_sents) ranking: defaultdict[int, float] = defaultdict(int) for i, sent in enumerate(word_tokenized_sents): diff --git a/pythainlp/summarize/keybert.py b/pythainlp/summarize/keybert.py index bae3ed07e..8ed84a57c 100644 --- a/pythainlp/summarize/keybert.py +++ b/pythainlp/summarize/keybert.py @@ -14,7 +14,7 @@ from collections import Counter from collections.abc import Iterable -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union from pythainlp.corpus import thai_stopwords from pythainlp.tokenize import word_tokenize @@ -24,9 +24,11 @@ class KeyBERT: + ft_pipeline: Any + def __init__( self, model_name: str = "airesearch/wangchanberta-base-att-spm-uncased" - ): + ) -> None: from transformers import pipeline self.ft_pipeline = pipeline( @@ -43,7 +45,7 @@ def extract_keywords( max_keywords: int = 5, min_df: int = 1, tokenizer: str = "newmm", - return_similarity=False, + return_similarity: bool = False, stop_words: Optional[Iterable[str]] = None, ) -> Union[list[str], list[tuple[str, float]]]: """Extract Thai keywords and/or keyphrases with KeyBERT algorithm. diff --git a/pythainlp/summarize/mt5.py b/pythainlp/summarize/mt5.py index f88597c5e..57101f5dd 100644 --- a/pythainlp/summarize/mt5.py +++ b/pythainlp/summarize/mt5.py @@ -18,7 +18,7 @@ def __init__( max_length: int = 100, skip_special_tokens: bool = True, pretrained_mt5_model_name: str = "", - ): + ) -> None: """Initialize mT5 Summarizer. :param str model_size: Size of the model ("small", "base", "large", @@ -50,14 +50,14 @@ def __init__( model_name = f"thanathorn/{CPE_KMUTT_THAI_SENTENCE_SUM}" else: model_name = pretrained_mt5_model_name - self.model_name = model_name - self.model = MT5ForConditionalGeneration.from_pretrained(model_name) - self.tokenizer = T5Tokenizer.from_pretrained(model_name) - self.num_beams = num_beams - self.no_repeat_ngram_size = no_repeat_ngram_size - self.min_length = min_length - self.max_length = max_length - self.skip_special_tokens = skip_special_tokens + self.model_name: str = model_name + self.model: MT5ForConditionalGeneration = MT5ForConditionalGeneration.from_pretrained(model_name) # type: ignore[assignment] + self.tokenizer: T5Tokenizer = T5Tokenizer.from_pretrained(model_name) # type: ignore[assignment] + self.num_beams: int = num_beams + self.no_repeat_ngram_size: int = no_repeat_ngram_size + self.min_length: int = min_length + self.max_length: int = max_length + self.skip_special_tokens: bool = skip_special_tokens def summarize(self, text: str) -> list[str]: preprocess_text = text.strip().replace("\n", "") diff --git a/pythainlp/tag/__init__.py b/pythainlp/tag/__init__.py index 23b02980c..585888c5d 100644 --- a/pythainlp/tag/__init__.py +++ b/pythainlp/tag/__init__.py @@ -7,7 +7,7 @@ such as its part-of-speech (POS) tag, and named entity (NE) tag. """ -__all__ = [ +__all__: list[str] = [ "PerceptronTagger", "NER", "NNER", diff --git a/pythainlp/tag/_tag_perceptron.py b/pythainlp/tag/_tag_perceptron.py index cbdc3faef..88b83e591 100644 --- a/pythainlp/tag/_tag_perceptron.py +++ b/pythainlp/tag/_tag_perceptron.py @@ -32,6 +32,12 @@ class AveragedPerceptron: http://honnibal.wordpress.com/2013/09/11/a-good-part-of-speechpos-tagger-in-about-200-lines-of-python/ """ + weights: dict[str, dict[str, float]] + classes: set[str] + _totals: dict[tuple[str, str], float] + _tstamps: dict[tuple[str, str], int] + i: int + def __init__(self) -> None: # Each feature gets its own weight vector, # so weights is a dict-of-dicts @@ -45,7 +51,7 @@ def __init__(self) -> None: # (tstamps is short for timestamps) self._tstamps: dict[tuple[str, str], int] = defaultdict(int) # Number of instances seen - self.i = 0 + self.i: int = 0 def predict(self, features: dict[str, float]) -> str: """Dot-product the features and current weights and return the best @@ -115,17 +121,21 @@ class PerceptronTagger: """ - START = ["-START-", "-START2-"] - END = ["-END-", "-END2-"] - AP_MODEL_LOC = "" + START: list[str] = ["-START-", "-START2-"] + END: list[str] = ["-END-", "-END2-"] + AP_MODEL_LOC: str = "" + + model: "AveragedPerceptron" + tagdict: dict[str, str] + classes: set[str] def __init__(self, path: str = "") -> None: """:param str path: model path""" - self.model = AveragedPerceptron() + self.model: "AveragedPerceptron" = AveragedPerceptron() self.tagdict: dict[str, str] = {} self.classes: set[str] = set() if path != "": - self.AP_MODEL_LOC = path + self.AP_MODEL_LOC: str = path self.load(self.AP_MODEL_LOC) def tag(self, tokens: Iterable[str]) -> list[tuple[str, str]]: diff --git a/pythainlp/tag/blackboard.py b/pythainlp/tag/blackboard.py index 42c43e2c3..d627da722 100644 --- a/pythainlp/tag/blackboard.py +++ b/pythainlp/tag/blackboard.py @@ -4,13 +4,13 @@ from __future__ import annotations # defined strings for special characters -CHAR_TO_ESCAPE = {" ": "_"} -ESCAPE_TO_CHAR = {v: k for k, v in CHAR_TO_ESCAPE.items()} +CHAR_TO_ESCAPE: dict[str, str] = {" ": "_"} +ESCAPE_TO_CHAR: dict[str, str] = {v: k for k, v in CHAR_TO_ESCAPE.items()} # map from Blackboard treebank POS tag to Universal POS tag # from Wannaphong Phatthiyaphaibun & Korakot Chaovavanich -TO_UD = { +TO_UD: dict[str, str] = { "": "", "AJ": "ADJ", "AV": "ADV", diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py index 0748593f7..73540bfac 100644 --- a/pythainlp/tag/crfchunk.py +++ b/pythainlp/tag/crfchunk.py @@ -72,15 +72,18 @@ class CRFchunk: garbage collected, though this is not guaranteed. """ + corpus: str _model_file_ctx: Optional[AbstractContextManager[Any]] + tagger: CRFTagger + xseq: list[dict[str, Any]] - def __init__(self, corpus: str = "orchidpp"): - self.corpus = corpus - self._model_file_ctx = None + def __init__(self, corpus: str = "orchidpp") -> None: + self.corpus: str = corpus + self._model_file_ctx: Optional[AbstractContextManager[Any]] = None self.load_model(self.corpus) def load_model(self, corpus: str) -> None: - self.tagger = CRFTagger() + self.tagger: CRFTagger = CRFTagger() if corpus == "orchidpp": corpus_files = files("pythainlp.corpus") model_file = corpus_files.joinpath("crfchunk_orchidpp.model") @@ -89,7 +92,7 @@ def load_model(self, corpus: str) -> None: self.tagger.open(str(model_path)) def parse(self, token_pos: list[tuple[str, str]]) -> list[str]: - self.xseq = extract_features(token_pos) + self.xseq: list[dict[str, Any]] = extract_features(token_pos) return self.tagger.tag(self.xseq) # type: ignore[no-any-return] def __enter__(self) -> CRFchunk: diff --git a/pythainlp/tag/named_entity.py b/pythainlp/tag/named_entity.py index 18ac7a8f2..a4e2bad84 100644 --- a/pythainlp/tag/named_entity.py +++ b/pythainlp/tag/named_entity.py @@ -30,13 +30,18 @@ class NER: The thai-nner engine supports nested NER and ignores corpus parameter. """ + name_engine: str + engine: Any + def __init__( self, engine: str = "thainer-v2", corpus: str = "thainer" ) -> None: + self.name_engine: str + self.engine: Any self.load_engine(engine=engine, corpus=corpus) def load_engine(self, engine: str, corpus: str) -> None: - self.name_engine = engine + self.name_engine: str = engine self.engine: Any = None # Engines that ignore corpus parameter @@ -123,13 +128,16 @@ class NNER: * *thai_nner* - Thai NER engine """ + engine: Any + def __init__(self, engine: str = "thai_nner") -> None: + self.engine: Any self.load_engine(engine) def load_engine(self, engine: str = "thai_nner") -> None: from pythainlp.tag.thai_nner import ThaiNNER - self.engine = ThaiNNER() + self.engine: Any = ThaiNNER() def tag(self, text: str, top_level_only: bool = False) -> tuple[list[str], list[dict[str, Any]]]: """This function tags nested named entities. diff --git a/pythainlp/tag/orchid.py b/pythainlp/tag/orchid.py index 3344e051a..261867b4a 100644 --- a/pythainlp/tag/orchid.py +++ b/pythainlp/tag/orchid.py @@ -7,7 +7,7 @@ # defined strings for special characters, # from Table 4 in ORCHID paper -CHAR_TO_ESCAPE = { +CHAR_TO_ESCAPE: dict[str, str] = { " ": "", "+": "", "-": "", @@ -32,11 +32,11 @@ ";": "", "/": "", } -ESCAPE_TO_CHAR = {v: k for k, v in CHAR_TO_ESCAPE.items()} +ESCAPE_TO_CHAR: dict[str, str] = {v: k for k, v in CHAR_TO_ESCAPE.items()} # map from ORCHID POS tag to Universal POS tag # from Korakot Chaovavanich -TO_UD = { +TO_UD: dict[str, str] = { "": "", # NOUN "NOUN": "NOUN", diff --git a/pythainlp/tag/perceptron.py b/pythainlp/tag/perceptron.py index fa2cfe436..98ebc0817 100644 --- a/pythainlp/tag/perceptron.py +++ b/pythainlp/tag/perceptron.py @@ -6,29 +6,30 @@ from __future__ import annotations import os +from typing import Optional from pythainlp.corpus import corpus_path, get_corpus_path from pythainlp.tag import PerceptronTagger, blackboard, orchid -_BLACKBOARD_NAME = "blackboard_pt_tagger" +_BLACKBOARD_NAME: str = "blackboard_pt_tagger" -_ORCHID_FILENAME = "pos_orchid_perceptron.json" -_ORCHID_PATH = os.path.join(corpus_path(), _ORCHID_FILENAME) +_ORCHID_FILENAME: str = "pos_orchid_perceptron.json" +_ORCHID_PATH: str = os.path.join(corpus_path(), _ORCHID_FILENAME) -_PUD_FILENAME = "pos_ud_perceptron-v0.2.json" -_PUD_PATH = os.path.join(corpus_path(), _PUD_FILENAME) +_PUD_FILENAME: str = "pos_ud_perceptron-v0.2.json" +_PUD_PATH: str = os.path.join(corpus_path(), _PUD_FILENAME) -_TDTB_FILENAME = "tdtb-pt_tagger.json" -_TDTB_PATH = os.path.join(corpus_path(), _TDTB_FILENAME) +_TDTB_FILENAME: str = "tdtb-pt_tagger.json" +_TDTB_PATH: str = os.path.join(corpus_path(), _TDTB_FILENAME) -_TUD_FILENAME = "pos_tud_perceptron.json" -_TUD_PATH = os.path.join(corpus_path(), _TUD_FILENAME) +_TUD_FILENAME: str = "pos_tud_perceptron.json" +_TUD_PATH: str = os.path.join(corpus_path(), _TUD_FILENAME) -_BLACKBOARD_TAGGER = None -_ORCHID_TAGGER = None -_PUD_TAGGER = None -_TDTB_TAGGER = None -_TUD_TAGGER = None +_BLACKBOARD_TAGGER: Optional[PerceptronTagger] = None +_ORCHID_TAGGER: Optional[PerceptronTagger] = None +_PUD_TAGGER: Optional[PerceptronTagger] = None +_TDTB_TAGGER: Optional[PerceptronTagger] = None +_TUD_TAGGER: Optional[PerceptronTagger] = None def _orchid_tagger() -> PerceptronTagger: diff --git a/pythainlp/tag/thai_nner.py b/pythainlp/tag/thai_nner.py index 1468dd1dd..1e94c205f 100644 --- a/pythainlp/tag/thai_nner.py +++ b/pythainlp/tag/thai_nner.py @@ -8,10 +8,13 @@ """ from __future__ import annotations -from typing import Optional, Union +from typing import TYPE_CHECKING, Optional, Union from pythainlp.corpus import get_corpus_path +if TYPE_CHECKING: + from thai_nner import NNER + __all__ = ["ThaiNNER"] @@ -37,7 +40,7 @@ def get_top_level_entities(entities: list[dict]) -> list[dict]: In nested NER, entities can contain other entities. This function filters the results to return only the outermost entities that are not contained - within any other entity. + within other entity. :param list[dict] entities: List of entity dictionaries with 'span', 'text', and 'entity_type' keys @@ -123,7 +126,7 @@ def __init__(self, path_model: Optional[str] = None) -> None: raise ImportError( "thai-nner library not found. Please install it with 'pip install thai-nner'." ) - self.model = NNER(path_model=path_model) + self.model: NNER = NNER(path_model=path_model) def tag(self, text: str, top_level_only: bool = False) -> tuple[list[str], list[dict]]: """Tag Thai text with nested named entities. diff --git a/pythainlp/tag/thainer.py b/pythainlp/tag/thainer.py index 5f775cb98..38a2391c9 100644 --- a/pythainlp/tag/thainer.py +++ b/pythainlp/tag/thainer.py @@ -8,13 +8,16 @@ __all__ = ["ThaiNameTagger"] -from typing import Union +from typing import TYPE_CHECKING, Union from pythainlp.corpus import get_corpus_path, thai_stopwords from pythainlp.tag.pos_tag import pos_tag from pythainlp.tokenize import word_tokenize from pythainlp.util import isthai +if TYPE_CHECKING: + from pycrfsuite import Tagger as CRFTagger + _TOKENIZER_ENGINE = "mm" @@ -22,7 +25,7 @@ def _is_stopword(word: str) -> bool: # เช็คว่าเป็นคำ return word in thai_stopwords() -def _doc2features(doc, i) -> dict: +def _doc2features(doc: list, i: int) -> dict: word = doc[i][0] postag = doc[i][1] @@ -89,6 +92,9 @@ class ThaiNameTagger: thainer14.get_ner("วันที่ 15 ก.ย. 61 ทดสอบระบบเวลา 14:49 น.") """ + crf: CRFTagger + pos_tag_name: str + def __init__(self, version: str = "1.4") -> None: """Thai named-entity recognizer. @@ -98,7 +104,7 @@ def __init__(self, version: str = "1.4") -> None: """ from pycrfsuite import Tagger as CRFTagger - self.crf = CRFTagger() + self.crf: "CRFTagger" = CRFTagger() if version == "1.4": model_path = get_corpus_path("thainer-1.4", version="1.4") @@ -109,7 +115,7 @@ def __init__(self, version: str = "1.4") -> None: " pythainlp.corpus.download('thainer-1.4')" ) self.crf.open(model_path) - self.pos_tag_name = "orchid_ud" + self.pos_tag_name: str = "orchid_ud" elif version == "1.5": model_path = get_corpus_path("thainer", version="1.5") if model_path is None: diff --git a/pythainlp/tag/unigram.py b/pythainlp/tag/unigram.py index 628bf7682..f98084b9f 100644 --- a/pythainlp/tag/unigram.py +++ b/pythainlp/tag/unigram.py @@ -7,32 +7,33 @@ import json import os +from typing import Any, Optional from pythainlp.corpus import corpus_path, get_corpus_path from pythainlp.tag import blackboard, orchid -_ORCHID_FILENAME = "pos_orchid_unigram.json" -_ORCHID_PATH = os.path.join(corpus_path(), _ORCHID_FILENAME) +_ORCHID_FILENAME: str = "pos_orchid_unigram.json" +_ORCHID_PATH: str = os.path.join(corpus_path(), _ORCHID_FILENAME) -_PUD_FILENAME = "pos_ud_unigram-v0.2.json" -_PUD_PATH = os.path.join(corpus_path(), _PUD_FILENAME) +_PUD_FILENAME: str = "pos_ud_unigram-v0.2.json" +_PUD_PATH: str = os.path.join(corpus_path(), _PUD_FILENAME) -_TDTB_FILENAME = "tdtb-unigram_tagger.json" -_TDTB_PATH = os.path.join(corpus_path(), _TDTB_FILENAME) +_TDTB_FILENAME: str = "tdtb-unigram_tagger.json" +_TDTB_PATH: str = os.path.join(corpus_path(), _TDTB_FILENAME) -_BLACKBOARD_NAME = "blackboard_unigram_tagger" +_BLACKBOARD_NAME: str = "blackboard_unigram_tagger" -_TUD_FILENAME = "pos_tud_unigram.json" -_TUD_PATH = os.path.join(corpus_path(), _TUD_FILENAME) +_TUD_FILENAME: str = "pos_tud_unigram.json" +_TUD_PATH: str = os.path.join(corpus_path(), _TUD_FILENAME) -_ORCHID_TAGGER = None -_PUD_TAGGER = None -_BLACKBOARD_TAGGER = None -_TDTB_TAGGER = None -_TUD_TAGGER = None +_ORCHID_TAGGER: Optional[dict[str, Any]] = None +_PUD_TAGGER: Optional[dict[str, Any]] = None +_BLACKBOARD_TAGGER: Optional[dict[str, Any]] = None +_TDTB_TAGGER: Optional[dict[str, Any]] = None +_TUD_TAGGER: Optional[dict[str, Any]] = None -def _orchid_tagger() -> dict: +def _orchid_tagger() -> dict[str, Any]: global _ORCHID_TAGGER if not _ORCHID_TAGGER: with open(_ORCHID_PATH, encoding="utf-8-sig") as fh: @@ -40,7 +41,7 @@ def _orchid_tagger() -> dict: return _ORCHID_TAGGER # type: ignore[no-any-return] -def _pud_tagger() -> dict: +def _pud_tagger() -> dict[str, Any]: global _PUD_TAGGER if not _PUD_TAGGER: with open(_PUD_PATH, encoding="utf-8-sig") as fh: @@ -48,7 +49,7 @@ def _pud_tagger() -> dict: return _PUD_TAGGER # type: ignore[no-any-return] -def _blackboard_tagger() -> dict: +def _blackboard_tagger() -> dict[str, Any]: global _BLACKBOARD_TAGGER if not _BLACKBOARD_TAGGER: path = get_corpus_path(_BLACKBOARD_NAME) @@ -59,7 +60,7 @@ def _blackboard_tagger() -> dict: return _BLACKBOARD_TAGGER # type: ignore[no-any-return] -def _thai_tdtb() -> dict: +def _thai_tdtb() -> dict[str, Any]: global _TDTB_TAGGER if not _TDTB_TAGGER: with open(_TDTB_PATH, encoding="utf-8-sig") as fh: @@ -67,7 +68,7 @@ def _thai_tdtb() -> dict: return _TDTB_TAGGER # type: ignore[no-any-return] -def _tud_tagger() -> dict: +def _tud_tagger() -> dict[str, Any]: global _TUD_TAGGER if not _TUD_TAGGER: with open(_TUD_PATH, encoding="utf-8-sig") as fh: @@ -76,7 +77,7 @@ def _tud_tagger() -> dict: def _find_tag( - words: list[str], dictdata: dict, default_tag: str = "" + words: list[str], dictdata: dict[str, Any], default_tag: str = "" ) -> list[tuple[str, str]]: keys = list(dictdata.keys()) return [ diff --git a/pythainlp/tag/wangchanberta_onnx.py b/pythainlp/tag/wangchanberta_onnx.py index bf2568698..eec491ea3 100644 --- a/pythainlp/tag/wangchanberta_onnx.py +++ b/pythainlp/tag/wangchanberta_onnx.py @@ -26,13 +26,13 @@ def __init__( SessionOptions, ) - self.model_name = model_name - self.model_version = model_version - self.options = SessionOptions() + self.model_name: str = model_name + self.model_version: str = model_version + self.options: SessionOptions = SessionOptions() self.options.graph_optimization_level = ( GraphOptimizationLevel.ORT_ENABLE_ALL ) - self.session = InferenceSession( + self.session: InferenceSession = InferenceSession( get_path_folder_corpus( self.model_name, self.model_version, file_onnx ), @@ -40,8 +40,8 @@ def __init__( providers=providers, ) self.session.disable_fallback() - self.outputs_name = self.session.get_outputs()[0].name - self.sp = spm.SentencePieceProcessor( + self.outputs_name: str = self.session.get_outputs()[0].name + self.sp: spm.SentencePieceProcessor = spm.SentencePieceProcessor( # type: ignore[assignment] model_file=get_path_folder_corpus( self.model_name, self.model_version, "sentencepiece.bpe.model" ) diff --git a/pythainlp/tokenize/__init__.py b/pythainlp/tokenize/__init__.py index 08c24fe29..cf17ccc90 100644 --- a/pythainlp/tokenize/__init__.py +++ b/pythainlp/tokenize/__init__.py @@ -5,7 +5,7 @@ from __future__ import annotations -__all__ = [ +__all__: list[str] = [ "thai2fit_tokenizer", "Tokenizer", "Trie", @@ -23,10 +23,10 @@ from pythainlp.corpus import thai_syllables, thai_words from pythainlp.util.trie import Trie -DEFAULT_WORD_TOKENIZE_ENGINE = "newmm" -DEFAULT_SENT_TOKENIZE_ENGINE = "crfcut" -DEFAULT_SUBWORD_TOKENIZE_ENGINE = "tcc" -DEFAULT_SYLLABLE_TOKENIZE_ENGINE = "han_solo" +DEFAULT_WORD_TOKENIZE_ENGINE: str = "newmm" +DEFAULT_SENT_TOKENIZE_ENGINE: str = "crfcut" +DEFAULT_SUBWORD_TOKENIZE_ENGINE: str = "tcc" +DEFAULT_SYLLABLE_TOKENIZE_ENGINE: str = "han_solo" @lru_cache diff --git a/pythainlp/tokenize/attacut.py b/pythainlp/tokenize/attacut.py index e215e1663..a5e5bf9ac 100644 --- a/pythainlp/tokenize/attacut.py +++ b/pythainlp/tokenize/attacut.py @@ -16,7 +16,10 @@ class AttacutTokenizer: - def __init__(self, model="attacut-sc"): + _MODEL_NAME: str + _tokenizer: Tokenizer + + def __init__(self, model: str = "attacut-sc") -> None: self._MODEL_NAME = "attacut-sc" if model == "attacut-c": diff --git a/pythainlp/tokenize/budoux.py b/pythainlp/tokenize/budoux.py index 2c6b95e1e..ffca64e97 100644 --- a/pythainlp/tokenize/budoux.py +++ b/pythainlp/tokenize/budoux.py @@ -13,13 +13,13 @@ from __future__ import annotations import threading -from typing import cast +from typing import Any, cast _parser = None _parser_lock = threading.Lock() -def _init_parser(): +def _init_parser() -> Any: """Lazy initialize and return a budoux parser instance. Raises ImportError when `budoux` is not installed, and RuntimeError diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index fbb8f9b14..c1a8606d5 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -928,7 +928,7 @@ def __init__( engine: str = "newmm", keep_whitespace: bool = True, join_broken_num: bool = True, - ): + ) -> None: """Initialize tokenizer object. :param str custom_dict: a file path, a list of vocaburaies* to be @@ -939,19 +939,19 @@ def __init__( :param bool keep_whitespace: True to keep whitespace, a common mark for end of phrase in Thai """ - self.__trie_dict = Trie([]) + self.__trie_dict: Trie = Trie([]) if custom_dict: self.__trie_dict = dict_trie(custom_dict) else: self.__trie_dict = word_dict_trie() - self.__engine = engine + self.__engine: str = engine if self.__engine not in ["newmm", "mm", "longest", "deepcut"]: raise NotImplementedError( "The Tokenizer class does not support " f"{self.__engine} for custom tokenizer." ) - self.__keep_whitespace = keep_whitespace - self.__join_broken_num = join_broken_num + self.__keep_whitespace: bool = keep_whitespace + self.__join_broken_num: bool = join_broken_num def word_tokenize(self, text: str) -> list[str]: """Main tokenization function. diff --git a/pythainlp/tokenize/etcc.py b/pythainlp/tokenize/etcc.py index 9e97f92bf..f756ac0ba 100644 --- a/pythainlp/tokenize/etcc.py +++ b/pythainlp/tokenize/etcc.py @@ -29,7 +29,7 @@ @lru_cache -def _cut_etcc(): +def _cut_etcc() -> "Tokenizer": """Lazy load ETCC tokenizer with cache""" return Tokenizer(get_corpus("etcc.txt"), engine="longest") diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py index 96a25eec7..821a8b546 100644 --- a/pythainlp/tokenize/han_solo.py +++ b/pythainlp/tokenize/han_solo.py @@ -48,6 +48,10 @@ def _get_tagger() -> pycrfsuite.Tagger: class Featurizer: # This class from ssg at https://github.com/ponrawee/ssg. + N: int + delimiter: Optional[str] + radius: int + def __init__( self, N: int = 2, diff --git a/pythainlp/tokenize/longest.py b/pythainlp/tokenize/longest.py index 684544830..a874a9239 100644 --- a/pythainlp/tokenize/longest.py +++ b/pythainlp/tokenize/longest.py @@ -46,7 +46,9 @@ class LongestMatchTokenizer: - def __init__(self, trie: Trie): + __trie: Trie + + def __init__(self, trie: Trie) -> None: self.__trie = trie @staticmethod diff --git a/pythainlp/tokenize/multi_cut.py b/pythainlp/tokenize/multi_cut.py index 2a3fec9e6..9018d62b4 100644 --- a/pythainlp/tokenize/multi_cut.py +++ b/pythainlp/tokenize/multi_cut.py @@ -25,18 +25,27 @@ class LatticeString(str): """String that keeps possible tokenizations""" - def __new__(cls, value, multi=None, in_dict=True): + unique: bool + multi: list[str] + in_dict: bool + + def __new__(cls, value: str, multi: Optional[list[str]] = None, in_dict: bool = True) -> "LatticeString": return str.__new__(cls, value) - def __init__(self, value, multi=None, in_dict=True): - self.unique = True + def __init__( + self, + value: str, + multi: Optional[list[str]] = None, + in_dict: bool = True, + ) -> None: + self.unique: bool = True if multi: - self.multi = list(multi) + self.multi: list[str] = list(multi) if len(self.multi) > 1: self.unique = False else: self.multi = [value] - self.in_dict = in_dict # if in dictionary + self.in_dict: bool = in_dict # if in dictionary _RE_NONTHAI = r"""(?x) @@ -59,7 +68,7 @@ def _multicut( list ) # main data structure - def serialize(p, p2): # helper function + def serialize(p: int, p2: int) -> Iterator[str]: # helper function for w in words_at[p]: p_ = p + len(w) if p_ == p2: diff --git a/pythainlp/tokenize/nercut.py b/pythainlp/tokenize/nercut.py index 0cf72d8a5..340ed228f 100644 --- a/pythainlp/tokenize/nercut.py +++ b/pythainlp/tokenize/nercut.py @@ -29,7 +29,7 @@ def segment( "DATE", "TIME", ], - tagger=_thainer, + tagger: NER = _thainer, ) -> list[str]: """Dictionary-based maximal matching word segmentation, constrained by Thai Character Cluster (TCC) boundaries, and combining tokens that are diff --git a/pythainlp/tokenize/nlpo3.py b/pythainlp/tokenize/nlpo3.py index add0d3281..7e25693cf 100644 --- a/pythainlp/tokenize/nlpo3.py +++ b/pythainlp/tokenize/nlpo3.py @@ -20,7 +20,7 @@ _load_lock = threading.Lock() # Thread safety for lazy loading -def _ensure_default_dict_loaded(): +def _ensure_default_dict_loaded() -> str: """Ensure the default dictionary is loaded. This function uses a lock to ensure thread-safe initialization. diff --git a/pythainlp/tokenize/thai2fit.py b/pythainlp/tokenize/thai2fit.py index fab9dac62..dcea0042e 100644 --- a/pythainlp/tokenize/thai2fit.py +++ b/pythainlp/tokenize/thai2fit.py @@ -9,7 +9,7 @@ @lru_cache -def thai2fit_tokenizer(): +def thai2fit_tokenizer() -> Tokenizer: """Lazy load Thai2Fit tokenizer with cache""" return Tokenizer( custom_dict=get_corpus("words_th_thai2fit_201810.txt"), engine="mm" diff --git a/pythainlp/tools/__init__.py b/pythainlp/tools/__init__.py index 3d6743c66..acc6502ff 100644 --- a/pythainlp/tools/__init__.py +++ b/pythainlp/tools/__init__.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -__all__ = [ +__all__: list[str] = [ "PYTHAINLP_DEFAULT_DATA_DIR", "get_full_data_path", "get_pythainlp_data_path", diff --git a/pythainlp/translate/core.py b/pythainlp/translate/core.py index 02c69606f..aebb74af8 100644 --- a/pythainlp/translate/core.py +++ b/pythainlp/translate/core.py @@ -59,10 +59,10 @@ def __init__( ZhThTranslator, ThFrTranslator, ] - self.engine = engine - self.src_lang = src_lang - self.use_gpu = use_gpu - self.target_lang = target_lang + self.engine: str = engine + self.src_lang: str = src_lang + self.use_gpu: bool = use_gpu + self.target_lang: str = target_lang self.load_model() def load_model(self) -> None: diff --git a/pythainlp/translate/en_th.py b/pythainlp/translate/en_th.py index 399c92561..c2997161f 100644 --- a/pythainlp/translate/en_th.py +++ b/pythainlp/translate/en_th.py @@ -66,13 +66,13 @@ class EnThTranslator: :param bool use_gpu : load model using GPU (Default is False) """ - def __init__(self, use_gpu: bool = False): - self._tokenizer = MosesTokenizer("en") + def __init__(self, use_gpu: bool = False) -> None: + self._tokenizer: MosesTokenizer = MosesTokenizer("en") - self._model_name = _EN_TH_MODEL_NAME + self._model_name: str = _EN_TH_MODEL_NAME _download_install(self._model_name) - self._model = TransformerModel.from_pretrained( + self._model: TransformerModel = TransformerModel.from_pretrained( # type: ignore[assignment] model_name_or_path=_get_translate_path( self._model_name, _EN_TH_FILE_NAME, @@ -122,8 +122,8 @@ class ThEnTranslator: :param bool use_gpu : load model using GPU (Default is False) """ - def __init__(self, use_gpu: bool = False): - self._model_name = _TH_EN_MODEL_NAME + def __init__(self, use_gpu: bool = False) -> None: + self._model_name: str = _TH_EN_MODEL_NAME _download_install(self._model_name) # Suppress model type mismatch warning from transformers @@ -133,7 +133,7 @@ def __init__(self, use_gpu: bool = False): "ignore", message="(?i).*using a model of type .* to instantiate a model of type.*", ) - self._model = TransformerModel.from_pretrained( + self._model: TransformerModel = TransformerModel.from_pretrained( # type: ignore[assignment] model_name_or_path=_get_translate_path( self._model_name, _TH_EN_FILE_NAME, diff --git a/pythainlp/translate/small100.py b/pythainlp/translate/small100.py index d24b33ecd..df69611fe 100644 --- a/pythainlp/translate/small100.py +++ b/pythainlp/translate/small100.py @@ -3,9 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from typing import Optional +from typing import TYPE_CHECKING, Optional -from transformers import M2M100ForConditionalGeneration +if TYPE_CHECKING: + from transformers import M2M100ForConditionalGeneration + import torch from .tokenization_small100 import SMALL100Tokenizer @@ -18,16 +20,24 @@ class Small100Translator: :param bool use_gpu : load model using GPU (Default is False) """ + pretrained: str + model: "M2M100ForConditionalGeneration" + tgt_lang: Optional[str] + tokenizer: "SMALL100Tokenizer" + translated: "torch.Tensor" + def __init__( self, use_gpu: bool = False, pretrained: str = "alirezamsh/small100", ) -> None: + from transformers import M2M100ForConditionalGeneration + self.pretrained = pretrained self.model = M2M100ForConditionalGeneration.from_pretrained( self.pretrained ) - self.tgt_lang: Optional[str] = None + self.tgt_lang = None if use_gpu: self.model = self.model.cuda() diff --git a/pythainlp/translate/th_fr.py b/pythainlp/translate/th_fr.py index 40eb84297..3c5c50e4e 100644 --- a/pythainlp/translate/th_fr.py +++ b/pythainlp/translate/th_fr.py @@ -14,6 +14,12 @@ from __future__ import annotations +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + import torch + class ThFrTranslator: """Thai-French Machine Translation @@ -29,6 +35,10 @@ class ThFrTranslator: :param bool use_gpu : load model using GPU (Default is False) """ + tokenizer_thfr: AutoTokenizer + model_thfr: AutoModelForSeq2SeqLM + translated: torch.Tensor + def __init__( self, use_gpu: bool = False, @@ -36,10 +46,10 @@ def __init__( ) -> None: from transformers import AutoModelForSeq2SeqLM, AutoTokenizer - self.tokenizer_thzh = AutoTokenizer.from_pretrained(pretrained) - self.model_thzh = AutoModelForSeq2SeqLM.from_pretrained(pretrained) + self.tokenizer_thfr = AutoTokenizer.from_pretrained(pretrained) + self.model_thfr = AutoModelForSeq2SeqLM.from_pretrained(pretrained) if use_gpu: - self.model_thzh = self.model_thzh.cuda() + self.model_thfr = self.model_thfr.cuda() def translate(self, text: str) -> str: """Translate text from Thai to French @@ -60,11 +70,11 @@ def translate(self, text: str) -> str: # output: "Test du système." """ - self.translated = self.model_thzh.generate( - **self.tokenizer_thzh(text, return_tensors="pt", padding=True) + self.translated = self.model_thfr.generate( + **self.tokenizer_thfr(text, return_tensors="pt", padding=True) ) decoded_list: list[str] = [ - self.tokenizer_thzh.decode(t, skip_special_tokens=True) + self.tokenizer_thfr.decode(t, skip_special_tokens=True) for t in self.translated ] return decoded_list[0] diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index 48623ed9d..86a2f9155 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -27,9 +27,11 @@ import os from pathlib import Path from shutil import copyfile -from typing import Any, Optional, Union, cast +from typing import TYPE_CHECKING, Optional, Union, cast + +if TYPE_CHECKING: + from sentencepiece import SentencePieceProcessor -import sentencepiece from transformers.tokenization_utils import BatchEncoding, PreTrainedTokenizer SPIECE_UNDERLINE = "▁" @@ -114,28 +116,44 @@ class SMALL100Tokenizer(PreTrainedTokenizer): """ - vocab_files_names = VOCAB_FILES_NAMES - max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES - pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP - model_input_names = ["input_ids", "attention_mask"] + vocab_files_names: dict[str, str] = VOCAB_FILES_NAMES + max_model_input_sizes: dict[str, int] = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + pretrained_vocab_files_map: dict[str, dict[str, str]] = PRETRAINED_VOCAB_FILES_MAP + model_input_names: list[str] = ["input_ids", "attention_mask"] prefix_tokens: Optional[list[int]] = [] suffix_tokens: list[int] = [] + sp_model_kwargs: dict[str, str] + language_codes: str + lang_code_to_token: dict[str, str] + vocab_file: str + encoder: dict[str, int] + decoder: dict[int, str] + spm_file: str + sp_model: SentencePieceProcessor + encoder_size: int + lang_token_to_id: dict[str, int] + lang_code_to_id: dict[str, int] + id_to_lang_token: dict[int, str] + _tgt_lang: str + cur_lang_id: int + num_madeup_words: int + def __init__( self, - vocab_file, - spm_file, - tgt_lang=None, - bos_token="", # noqa: S107 - eos_token="", # noqa: S107 - sep_token="", # noqa: S107 - pad_token="", # noqa: S107 - unk_token="", # noqa: S107 - language_codes="m2m100", - sp_model_kwargs: Optional[dict[str, Any]] = None, - num_madeup_words=8, - **kwargs, + vocab_file: str, + spm_file: str, + tgt_lang: Optional[str] = None, + bos_token: str = "", # noqa: S107 + eos_token: str = "", # noqa: S107 + sep_token: str = "", # noqa: S107 + pad_token: str = "", # noqa: S107 + unk_token: str = "", # noqa: S107 + language_codes: str = "m2m100", + sp_model_kwargs: Optional[dict[str, str]] = None, + num_madeup_words: int = 8, + **kwargs: str, ) -> None: self.sp_model_kwargs = ( {} if sp_model_kwargs is None else sp_model_kwargs @@ -175,10 +193,10 @@ def __init__( encoder_data = load_json(vocab_file) if not isinstance(encoder_data, dict): raise ValueError("encoder must be a dict") - self.encoder: dict[str, int] = cast(dict[str, int], encoder_data) + self.encoder = cast(dict[str, int], encoder_data) self.decoder = {v: k for k, v in self.encoder.items()} self.spm_file = spm_file - self.sp_model = load_spm(spm_file, self.sp_model_kwargs) + self.sp_model = load_spm(spm_file, self.sp_model_kwargs) # SentencePieceProcessor self.encoder_size = len(self.encoder) @@ -203,7 +221,7 @@ def __init__( @property def vocab_size(self) -> int: # Type ignore for external library dict operations - return len(self.encoder) + len(self.lang_token_to_id) + self.num_madeup_words # type: ignore[no-any-return] + return len(self.encoder) + len(self.lang_token_to_id) + self.num_madeup_words @property def tgt_lang(self) -> str: @@ -375,15 +393,15 @@ def prepare_seq2seq_batch( src_texts: list[str], tgt_texts: Optional[list[str]] = None, tgt_lang: str = "ro", - **kwargs, + **kwargs: Any, ) -> BatchEncoding: self.tgt_lang = tgt_lang self.set_lang_special_tokens(self.tgt_lang) return super().prepare_seq2seq_batch(src_texts, tgt_texts, **kwargs) def _build_translation_inputs( - self, raw_inputs, tgt_lang: Optional[str], **extra_kwargs - ): + self, raw_inputs: Union[str, list[str]], tgt_lang: Optional[str], **extra_kwargs: str + ) -> dict[str, Any]: """Used by translation pipeline, to prepare inputs for the generate function""" if tgt_lang is None: @@ -392,12 +410,12 @@ def _build_translation_inputs( ) self.tgt_lang = tgt_lang inputs = self(raw_inputs, add_special_tokens=True, **extra_kwargs) - return inputs + return inputs # type: ignore[no-any-return] - def _switch_to_input_mode(self): + def _switch_to_input_mode(self) -> None: self.set_lang_special_tokens(self.tgt_lang) - def _switch_to_target_mode(self): + def _switch_to_target_mode(self) -> None: self.prefix_tokens = None self.suffix_tokens = [self.eos_token_id] @@ -418,18 +436,20 @@ def get_lang_id(self, lang: str) -> int: def load_spm( - path: str, sp_model_kwargs: dict[str, Any] -) -> sentencepiece.SentencePieceProcessor: + path: str, sp_model_kwargs: dict[str, str] +) -> SentencePieceProcessor: + import sentencepiece + spm = sentencepiece.SentencePieceProcessor(**sp_model_kwargs) spm.Load(str(path)) return spm -def load_json(path: str) -> Union[dict[Any, Any], list[Any]]: +def load_json(path: str) -> Union[dict[str, str], list[str]]: with open(path) as f: return json.load(f) # type: ignore[no-any-return] -def save_json(data, path: str) -> None: +def save_json(data: Union[dict[str, str], list[str]], path: str) -> None: with open(path, "w") as f: json.dump(data, f, indent=2) diff --git a/pythainlp/translate/zh_th.py b/pythainlp/translate/zh_th.py index 898be6c72..52a331d98 100644 --- a/pythainlp/translate/zh_th.py +++ b/pythainlp/translate/zh_th.py @@ -11,6 +11,12 @@ from __future__ import annotations +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + import torch + class ThZhTranslator: """Thai-Chinese Machine Translation @@ -23,6 +29,10 @@ class ThZhTranslator: :param bool use_gpu : load model using GPU (Default is False) """ + tokenizer_thzh: AutoTokenizer + model_thzh: AutoModelForSeq2SeqLM + translated: torch.Tensor + def __init__( self, use_gpu: bool = False, @@ -75,6 +85,10 @@ class ZhThTranslator: :param bool use_gpu : load model using GPU (Default is False) """ + tokenizer_zhth: AutoTokenizer + model_zhth: AutoModelForSeq2SeqLM + translated: torch.Tensor + def __init__( self, use_gpu: bool = False, diff --git a/pythainlp/transliterate/__init__.py b/pythainlp/transliterate/__init__.py index ecd556e05..60a6bcf21 100644 --- a/pythainlp/transliterate/__init__.py +++ b/pythainlp/transliterate/__init__.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 """Transliteration.""" -__all__ = [ +__all__: list[str] = [ "pronunciate", "puan", "romanize", diff --git a/pythainlp/transliterate/core.py b/pythainlp/transliterate/core.py index 7c2714916..cd01bad32 100644 --- a/pythainlp/transliterate/core.py +++ b/pythainlp/transliterate/core.py @@ -5,9 +5,9 @@ from typing import Callable -DEFAULT_ROMANIZE_ENGINE = "royin" -DEFAULT_TRANSLITERATE_ENGINE = "thaig2p" -DEFAULT_PRONUNCIATE_ENGINE = "w2p" +DEFAULT_ROMANIZE_ENGINE: str = "royin" +DEFAULT_TRANSLITERATE_ENGINE: str = "thaig2p" +DEFAULT_PRONUNCIATE_ENGINE: str = "w2p" def romanize( diff --git a/pythainlp/transliterate/ipa.py b/pythainlp/transliterate/ipa.py index a18c5014e..a4e8d026b 100644 --- a/pythainlp/transliterate/ipa.py +++ b/pythainlp/transliterate/ipa.py @@ -11,9 +11,11 @@ from __future__ import annotations +from typing import Any + import epitran -_EPI_THA = epitran.Epitran("tha-Thai") +_EPI_THA: Any = epitran.Epitran("tha-Thai") def transliterate(text: str) -> str: diff --git a/pythainlp/transliterate/iso_11940.py b/pythainlp/transliterate/iso_11940.py index e3a544235..1c1b1383e 100644 --- a/pythainlp/transliterate/iso_11940.py +++ b/pythainlp/transliterate/iso_11940.py @@ -10,7 +10,7 @@ from __future__ import annotations -_consonants = { +_consonants: dict[str, str] = { "ก": "k", "ข": "k̄h", "ฃ": "ḳ̄h", @@ -59,7 +59,7 @@ "ฮ": "ḥ", } -_vowels = { +_vowels: dict[str, str] = { "ะ": "a", "ั": "ạ", "า": "ā", @@ -84,7 +84,7 @@ "อ": "x", } -_tone_marks = { +_tone_marks: dict[str, str] = { "่": "–̀".replace("–", ""), "้": "–̂".replace("–", ""), "๊": "–́".replace("–", ""), @@ -96,7 +96,7 @@ "–ฺ".replace("–", ""): "–̥".replace("–", ""), } -_punctuation_and_digits = { +_punctuation_and_digits: dict[str, str] = { # ฯ can has two meanings in ISO 11940. # If it is for abbreviation, it is paiyan noi. # If it is for sentence termination, it is angkhan diao. diff --git a/pythainlp/transliterate/royin.py b/pythainlp/transliterate/royin.py index 72e17a869..635be5096 100644 --- a/pythainlp/transliterate/royin.py +++ b/pythainlp/transliterate/royin.py @@ -16,10 +16,10 @@ from pythainlp import thai_consonants, word_tokenize # Romanized vowels for checking -_ROMANIZED_VOWELS = "aeiou" +_ROMANIZED_VOWELS: str = "aeiou" # vowel -_vowel_patterns = """เ*ียว,\\1iao +_vowel_patterns: str = """เ*ียว,\\1iao แ*็ว,\\1aeo เ*ือย,\\1ueai แ*ว,\\1aeo @@ -74,10 +74,10 @@ _vowel_patterns = _vowel_patterns.replace("#", "([คนพมห])") _vowel_patterns = _vowel_patterns.replace("$", "([กตทปศส])") -_VOWELS = [x.split(",") for x in _vowel_patterns.split("\n")] +_VOWELS: list[list[str]] = [x.split(",") for x in _vowel_patterns.split("\n")] # พยัญชนะ ต้น สะกด -_CONSONANTS = { +_CONSONANTS: dict[str, list[str]] = { "ก": ["k", "k"], "ข": ["kh", "k"], "ฃ": ["kh", "k"], @@ -126,9 +126,9 @@ "ฮ": ["h", ""], } -_THANTHAKHAT = "\u0e4c" -_RE_CONSONANT = re.compile(f"[{thai_consonants}]") -_RE_NORMALIZE = re.compile( +_THANTHAKHAT: str = "\u0e4c" +_RE_CONSONANT: re.Pattern[str] = re.compile(f"[{thai_consonants}]") +_RE_NORMALIZE: re.Pattern[str] = re.compile( f"จน์|มณ์|ณฑ์|ทร์|ตร์|[{thai_consonants}]{_THANTHAKHAT}|" f"[{thai_consonants}][\u0e30-\u0e39]{_THANTHAKHAT}" # Paiyannoi, Maiyamok, Tonemarks, Thanthakhat, Nikhahit, other signs diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index 9480ee1b3..caa06916c 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -6,6 +6,7 @@ from __future__ import annotations import random +from typing import TYPE_CHECKING, Optional, Tuple import torch import torch.nn.functional as F @@ -13,41 +14,54 @@ from pythainlp.corpus import get_corpus_path +if TYPE_CHECKING: + from typing import Dict + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") _MODEL_NAME = "thai2rom-pytorch-attn" class ThaiTransliterator: - def __init__(self): + __model_filename: str + _maxlength: int + _char_to_ix: Dict[str, int] + _ix_to_char: Dict[int, str] + _target_char_to_ix: Dict[str, int] + _ix_to_target_char: Dict[int, str] + _encoder: "Encoder" + _decoder: "AttentionDecoder" + _network: "Seq2Seq" + + def __init__(self) -> None: """Transliteration of Thai words. Now supports Thai to Latin (romanization) """ # get the model, download it if it's not available locally - self.__model_filename = get_corpus_path(_MODEL_NAME) + self.__model_filename: str = get_corpus_path(_MODEL_NAME) # type: ignore[assignment] loader = torch.load(self.__model_filename, map_location=device) INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT = loader["encoder_params"] OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT = loader["decoder_params"] - self._maxlength = 100 + self._maxlength: int = 100 - self._char_to_ix = loader["char_to_ix"] - self._ix_to_char = loader["ix_to_char"] - self._target_char_to_ix = loader["target_char_to_ix"] - self._ix_to_target_char = loader["ix_to_target_char"] + self._char_to_ix: "Dict[str, int]" = loader["char_to_ix"] + self._ix_to_char: "Dict[int, str]" = loader["ix_to_char"] + self._target_char_to_ix: "Dict[str, int]" = loader["target_char_to_ix"] + self._ix_to_target_char: "Dict[int, str]" = loader["ix_to_target_char"] # encoder/ decoder # Restore the model and construct the encoder and decoder. - self._encoder = Encoder(INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT) + self._encoder: "Encoder" = Encoder(INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT) - self._decoder = AttentionDecoder( + self._decoder: "AttentionDecoder" = AttentionDecoder( OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT ) - self._network = Seq2Seq( + self._network: "Seq2Seq" = Seq2Seq( self._encoder, self._decoder, self._target_char_to_ix[""], @@ -58,7 +72,7 @@ def __init__(self): self._network.load_state_dict(loader["model_state_dict"]) self._network.eval() - def _prepare_sequence_in(self, text: str): + def _prepare_sequence_in(self, text: str) -> torch.Tensor: """Prepare input sequence for PyTorch""" idxs = [] for ch in text: @@ -99,8 +113,12 @@ def romanize(self, text: str) -> str: class Encoder(nn.Module): def __init__( - self, vocabulary_size, embedding_size, hidden_size, dropout=0.5 - ): + self, + vocabulary_size: int, + embedding_size: int, + hidden_size: int, + dropout: float = 0.5, + ) -> None: """Constructor""" super().__init__() self.hidden_size = hidden_size @@ -116,7 +134,9 @@ def __init__( self.dropout = nn.Dropout(dropout) - def forward(self, sequences, sequences_lengths): + def forward( + self, sequences: torch.Tensor, sequences_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: # sequences: (batch_size, sequence_length=MAX_LENGTH) # sequences_lengths: (batch_size) @@ -150,7 +170,7 @@ def forward(self, sequences, sequences_lengths): ) return sequences_output, hidden - def init_hidden(self, batch_size): + def init_hidden(self, batch_size: int) -> Tuple[torch.Tensor, torch.Tensor]: h_0 = torch.zeros( [2, batch_size, self.hidden_size // 2], requires_grad=True ).to(device) @@ -162,11 +182,13 @@ def init_hidden(self, batch_size): class Attn(nn.Module): - def __init__(self, method, hidden_size): + def __init__(self, method: str, hidden_size: int) -> None: super().__init__() self.method = method self.hidden_size = hidden_size + self.attn: nn.Linear + self.other: nn.Parameter if self.method == "general": self.attn = nn.Linear(self.hidden_size, hidden_size) @@ -175,7 +197,12 @@ def __init__(self, method, hidden_size): self.attn = nn.Linear(self.hidden_size * 2, hidden_size) self.other = nn.Parameter(torch.FloatTensor(1, hidden_size)) - def forward(self, hidden, encoder_outputs, mask): + def forward( + self, + hidden: torch.Tensor, + encoder_outputs: torch.Tensor, + mask: torch.Tensor, + ) -> torch.Tensor: # Calculate energies for each encoder output if self.method == "dot": attn_energies = torch.bmm( @@ -209,8 +236,12 @@ def forward(self, hidden, encoder_outputs, mask): class AttentionDecoder(nn.Module): def __init__( - self, vocabulary_size, embedding_size, hidden_size, dropout=0.5 - ): + self, + vocabulary_size: int, + embedding_size: int, + hidden_size: int, + dropout: float = 0.5, + ) -> None: """Constructor""" super().__init__() self.vocabulary_size = vocabulary_size @@ -230,7 +261,13 @@ def __init__( self.dropout = nn.Dropout(dropout) - def forward(self, input_character, last_hidden, encoder_outputs, mask): + def forward( + self, + input_character: torch.Tensor, + last_hidden: torch.Tensor, + encoder_outputs: torch.Tensor, + mask: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Defines the forward computation of the decoder""" # input_character: (batch_size, 1) # last_hidden: (batch_size, hidden_dim) @@ -260,12 +297,12 @@ def forward(self, input_character, last_hidden, encoder_outputs, mask): class Seq2Seq(nn.Module): def __init__( self, - encoder, - decoder, - target_start_token, - target_end_token, - max_length, - ): + encoder: Encoder, + decoder: AttentionDecoder, + target_start_token: int, + target_end_token: int, + max_length: int, + ) -> None: super().__init__() self.encoder = encoder @@ -281,13 +318,17 @@ def __init__( f"Got encoder={encoder.hidden_size}, decoder={decoder.hidden_size}" ) - def create_mask(self, source_seq): + def create_mask(self, source_seq: torch.Tensor) -> torch.Tensor: mask = source_seq != self.pad_idx return mask def forward( - self, source_seq, source_seq_len, target_seq, teacher_forcing_ratio=0.5 - ): + self, + source_seq: torch.Tensor, + source_seq_len: torch.Tensor, + target_seq: Optional[torch.Tensor], + teacher_forcing_ratio: float = 0.5, + ) -> torch.Tensor: # source_seq: (batch_size, MAX_LENGTH) # source_seq_len: (batch_size, 1) # target_seq: (batch_size, MAX_LENGTH) diff --git a/pythainlp/transliterate/thai2rom_onnx.py b/pythainlp/transliterate/thai2rom_onnx.py index e986afb9c..d255abc2e 100644 --- a/pythainlp/transliterate/thai2rom_onnx.py +++ b/pythainlp/transliterate/thai2rom_onnx.py @@ -6,27 +6,31 @@ from __future__ import annotations import json +from typing import TYPE_CHECKING import numpy as np from onnxruntime import InferenceSession from pythainlp.corpus import get_corpus_path +if TYPE_CHECKING: + from typing import Dict, List + _MODEL_ENCODER_NAME = "thai2rom_encoder_onnx" _MODEL_DECODER_NAME = "thai2rom_decoder_onnx" _MODEL_CONFIG_NAME = "thai2rom_config_onnx" class ThaiTransliterator_ONNX: - def __init__(self): + def __init__(self) -> None: """Transliteration of Thai words. Now supports Thai to Latin (romanization) """ # get the model, download it if it's not available locally - self.__encoder_filename = get_corpus_path(_MODEL_ENCODER_NAME) - self.__decoder_filename = get_corpus_path(_MODEL_DECODER_NAME) - self.__config_filename = get_corpus_path(_MODEL_CONFIG_NAME) + self.__encoder_filename: str = get_corpus_path(_MODEL_ENCODER_NAME) # type: ignore[assignment] + self.__decoder_filename: str = get_corpus_path(_MODEL_DECODER_NAME) # type: ignore[assignment] + self.__config_filename: str = get_corpus_path(_MODEL_CONFIG_NAME) # type: ignore[assignment] # loader = torch.load(self.__model_filename, map_location=device) with open(str(self.__config_filename)) as f: @@ -34,20 +38,20 @@ def __init__(self): OUTPUT_DIM = loader["output_dim"] - self._maxlength = 100 + self._maxlength: int = 100 - self._char_to_ix = loader["char_to_ix"] - self._ix_to_char = loader["ix_to_char"] - self._target_char_to_ix = loader["target_char_to_ix"] - self._ix_to_target_char = loader["ix_to_target_char"] + self._char_to_ix: Dict[str, int] = loader["char_to_ix"] + self._ix_to_char: Dict[int, str] = loader["ix_to_char"] + self._target_char_to_ix: Dict[str, int] = loader["target_char_to_ix"] + self._ix_to_target_char: Dict[int, str] = loader["ix_to_target_char"] # encoder/ decoder # Load encoder decoder onnx models. - self._encoder = InferenceSession(self.__encoder_filename) + self._encoder: InferenceSession = InferenceSession(self.__encoder_filename) - self._decoder = InferenceSession(self.__decoder_filename) + self._decoder: InferenceSession = InferenceSession(self.__decoder_filename) - self._network = Seq2Seq_ONNX( + self._network: Seq2Seq_ONNX = Seq2Seq_ONNX( self._encoder, self._decoder, self._target_char_to_ix[""], @@ -56,7 +60,7 @@ def __init__(self): target_vocab_size=OUTPUT_DIM, ) - def _prepare_sequence_in(self, text: str): + def _prepare_sequence_in(self, text: str) -> np.ndarray: """Prepare input sequence for ONNX""" idxs = [] for ch in text: @@ -90,13 +94,13 @@ def romanize(self, text: str) -> str: class Seq2Seq_ONNX: def __init__( self, - encoder, - decoder, - target_start_token, - target_end_token, - max_length, - target_vocab_size, - ): + encoder: InferenceSession, + decoder: InferenceSession, + target_start_token: int, + target_end_token: int, + max_length: int, + target_vocab_size: int, + ) -> None: super().__init__() self.encoder = encoder @@ -108,11 +112,11 @@ def __init__( self.target_vocab_size = target_vocab_size - def create_mask(self, source_seq): + def create_mask(self, source_seq: np.ndarray) -> np.ndarray: mask = source_seq != self.pad_idx return mask - def run(self, source_seq, source_seq_len): + def run(self, source_seq: np.ndarray, source_seq_len: List[int]) -> np.ndarray: # source_seq: (batch_size, MAX_LENGTH) # source_seq_len: (batch_size, 1) # target_seq: (batch_size, MAX_LENGTH) diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index b7cb3323f..3b0f82458 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -8,6 +8,7 @@ from __future__ import annotations import random +from typing import TYPE_CHECKING, Optional, Union import numpy as np import torch @@ -16,6 +17,9 @@ from pythainlp.corpus import get_corpus_path +if TYPE_CHECKING: + from numpy.typing import NDArray + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") _MODEL_NAME = "thai-g2p" @@ -33,31 +37,41 @@ class ThaiG2P: https://github.com/wannaphong/thai-g2p """ - def __init__(self): + __model_filename: str + _maxlength: int + _char_to_ix: dict[str, int] + _ix_to_char: dict[int, str] + _target_char_to_ix: dict[str, int] + _ix_to_target_char: dict[int, str] + _encoder: "Encoder" + _decoder: "AttentionDecoder" + _network: "Seq2Seq" + + def __init__(self) -> None: # get the model, download it if it's not available locally - self.__model_filename = get_corpus_path(_MODEL_NAME) + self.__model_filename: str = get_corpus_path(_MODEL_NAME) # type: ignore[assignment] loader = torch.load(self.__model_filename, map_location=device) INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT = loader["encoder_params"] OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT = loader["decoder_params"] - self._maxlength = 100 + self._maxlength: int = 100 - self._char_to_ix = loader["char_to_ix"] - self._ix_to_char = loader["ix_to_char"] - self._target_char_to_ix = loader["target_char_to_ix"] - self._ix_to_target_char = loader["ix_to_target_char"] + self._char_to_ix: dict[str, int] = loader["char_to_ix"] + self._ix_to_char: dict[int, str] = loader["ix_to_char"] + self._target_char_to_ix: dict[str, int] = loader["target_char_to_ix"] + self._ix_to_target_char: dict[int, str] = loader["ix_to_target_char"] # encoder/ decoder # Restore the model and construct the encoder and decoder. - self._encoder = Encoder(INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT) + self._encoder: "Encoder" = Encoder(INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT) - self._decoder = AttentionDecoder( + self._decoder: "AttentionDecoder" = AttentionDecoder( OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT ) - self._network = Seq2Seq( + self._network: "Seq2Seq" = Seq2Seq( self._encoder, self._decoder, self._target_char_to_ix[""], @@ -68,7 +82,7 @@ def __init__(self): self._network.load_state_dict(loader["model_state_dict"]) self._network.eval() - def _prepare_sequence_in(self, text: str): + def _prepare_sequence_in(self, text: str) -> torch.Tensor: """Prepare input sequence for PyTorch.""" idxs = [] for ch in text: @@ -110,8 +124,12 @@ def g2p(self, text: str) -> str: class Encoder(nn.Module): def __init__( - self, vocabulary_size, embedding_size, hidden_size, dropout=0.5 - ): + self, + vocabulary_size: int, + embedding_size: int, + hidden_size: int, + dropout: float = 0.5, + ) -> None: """Constructor""" super().__init__() self.hidden_size = hidden_size @@ -127,12 +145,16 @@ def __init__( self.dropout = nn.Dropout(dropout) - def forward(self, sequences, sequences_lengths): + def forward( + self, + sequences: torch.Tensor, + sequences_lengths: Union[NDArray, list[int]], + ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]: # sequences: (batch_size, sequence_length=MAX_LENGTH) # sequences_lengths: (batch_size) batch_size = sequences.size(0) - self.hidden = self.init_hidden(batch_size) + self.hidden: tuple[torch.Tensor, torch.Tensor] = self.init_hidden(batch_size) sequences_lengths = np.sort(sequences_lengths)[::-1] index_sorted = np.argsort( @@ -163,7 +185,9 @@ def forward(self, sequences, sequences_lengths): return sequences_output, self.hidden - def init_hidden(self, batch_size): + def init_hidden( + self, batch_size: int + ) -> tuple[torch.Tensor, torch.Tensor]: h_0 = torch.zeros( [2, batch_size, self.hidden_size // 2], requires_grad=True ).to(device) @@ -175,20 +199,27 @@ def init_hidden(self, batch_size): class Attn(nn.Module): - def __init__(self, method, hidden_size): + def __init__(self, method: str, hidden_size: int) -> None: super().__init__() self.method = method self.hidden_size = hidden_size if self.method == "general": - self.attn = nn.Linear(self.hidden_size, hidden_size) + self.attn: nn.Linear = nn.Linear(self.hidden_size, hidden_size) elif self.method == "concat": self.attn = nn.Linear(self.hidden_size * 2, hidden_size) - self.other = nn.Parameter(torch.FloatTensor(1, hidden_size)) + self.other: nn.Parameter = nn.Parameter( + torch.FloatTensor(1, hidden_size) + ) - def forward(self, hidden, encoder_outputs, mask): + def forward( + self, + hidden: torch.Tensor, + encoder_outputs: torch.Tensor, + mask: torch.Tensor, + ) -> torch.Tensor: # Calculate energies for each encoder output if self.method == "dot": attn_energies = torch.bmm( @@ -222,8 +253,12 @@ def forward(self, hidden, encoder_outputs, mask): class AttentionDecoder(nn.Module): def __init__( - self, vocabulary_size, embedding_size, hidden_size, dropout=0.5 - ): + self, + vocabulary_size: int, + embedding_size: int, + hidden_size: int, + dropout: float = 0.5, + ) -> None: """Constructor""" super().__init__() self.vocabulary_size = vocabulary_size @@ -243,7 +278,13 @@ def __init__( self.dropout = nn.Dropout(dropout) - def forward(self, input_character, last_hidden, encoder_outputs, mask): + def forward( + self, + input_character: torch.Tensor, + last_hidden: torch.Tensor, + encoder_outputs: torch.Tensor, + mask: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ "Defines the forward computation of the decoder""" # input_character: (batch_size, 1) # last_hidden: (batch_size, hidden_dim) @@ -273,12 +314,12 @@ def forward(self, input_character, last_hidden, encoder_outputs, mask): class Seq2Seq(nn.Module): def __init__( self, - encoder, - decoder, - target_start_token, - target_end_token, - max_length, - ): + encoder: Encoder, + decoder: AttentionDecoder, + target_start_token: int, + target_end_token: int, + max_length: int, + ) -> None: super().__init__() self.encoder = encoder @@ -294,13 +335,17 @@ def __init__( f"Got encoder={encoder.hidden_size}, decoder={decoder.hidden_size}" ) - def create_mask(self, source_seq): + def create_mask(self, source_seq: torch.Tensor) -> torch.Tensor: mask = source_seq != self.pad_idx return mask def forward( - self, source_seq, source_seq_len, target_seq, teacher_forcing_ratio=0.5 - ): + self, + source_seq: torch.Tensor, + source_seq_len: Union[NDArray, list[int]], + target_seq: Optional[torch.Tensor], + teacher_forcing_ratio: float = 0.5, + ) -> torch.Tensor: # source_seq: (batch_size, MAX_LENGTH) # source_seq_len: (batch_size, 1) # target_seq: (batch_size, MAX_LENGTH) @@ -355,7 +400,7 @@ def forward( decoder_input = ( target_seq[:, di].reshape(batch_size, 1) - if teacher_force + if teacher_force and target_seq is not None else topi.detach() ) diff --git a/pythainlp/transliterate/thaig2p_v2.py b/pythainlp/transliterate/thaig2p_v2.py index 7300b959d..5e663fc50 100644 --- a/pythainlp/transliterate/thaig2p_v2.py +++ b/pythainlp/transliterate/thaig2p_v2.py @@ -9,7 +9,10 @@ # Use a pipeline as a high-level helper from __future__ import annotations -from transformers import pipeline +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from transformers import Pipeline class ThaiG2P: @@ -24,8 +27,12 @@ class ThaiG2P: https://huggingface.co/pythainlp/thaig2p-v2.0 """ - def __init__(self, device: str = "cpu"): - self.pipe = pipeline( + pipe: Pipeline + + def __init__(self, device: str = "cpu") -> None: + from transformers import pipeline + + self.pipe: "Pipeline" = pipeline( "text2text-generation", model="pythainlp/thaig2p-v2.0", device=device, @@ -38,7 +45,7 @@ def g2p(self, text: str) -> str: _THAI_G2P = None -def transliterate(text: str, device="cpu") -> str: +def transliterate(text: str, device: str = "cpu") -> str: global _THAI_G2P if _THAI_G2P is None: _THAI_G2P = ThaiG2P(device=device) diff --git a/pythainlp/transliterate/umt5_thaig2p.py b/pythainlp/transliterate/umt5_thaig2p.py index e3831d60b..aec751187 100644 --- a/pythainlp/transliterate/umt5_thaig2p.py +++ b/pythainlp/transliterate/umt5_thaig2p.py @@ -9,7 +9,10 @@ # Use a pipeline as a high-level helper from __future__ import annotations -from transformers import pipeline +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from transformers import Pipeline class Umt5ThaiG2P: @@ -24,7 +27,11 @@ class Umt5ThaiG2P: https://huggingface.co/B-K/umt5-thai-g2p-v2-0.5k """ - def __init__(self, device: str = "cpu"): + pipe: Pipeline + + def __init__(self, device: str = "cpu") -> None: + from transformers import pipeline + self.pipe = pipeline( "text2text-generation", model="B-K/umt5-thai-g2p-v2-0.5k", @@ -38,7 +45,7 @@ def g2p(self, text: str) -> str: _THAI_G2P = None -def transliterate(text: str, device="cpu") -> str: +def transliterate(text: str, device: str = "cpu") -> str: global _THAI_G2P if _THAI_G2P is None: _THAI_G2P = Umt5ThaiG2P(device=device) diff --git a/pythainlp/transliterate/w2p.py b/pythainlp/transliterate/w2p.py index d9ac1650e..e713a412d 100644 --- a/pythainlp/transliterate/w2p.py +++ b/pythainlp/transliterate/w2p.py @@ -7,12 +7,15 @@ from __future__ import annotations -from typing import Optional +from typing import TYPE_CHECKING, Optional import numpy as np from pythainlp.corpus import download, get_corpus_path +if TYPE_CHECKING: + from numpy.typing import NDArray + _GRAPHEMES = list( "พจใงต้ืฮแาฐฒฤๅูศฅถฺฎหคสุขเึดฟำฝยลอ็ม" + " ณิฑชฉซทรฏฬํัฃวก่ป์ผฆบี๊ธญฌษะไ๋นโภ?" ) @@ -24,21 +27,21 @@ class _Hparams: - batch_size = 256 - enc_maxlen = 30 * 2 - dec_maxlen = 40 * 2 - num_epochs = 50 * 2 - hidden_units = 64 * 8 - emb_units = 64 * 4 - graphemes = ["", "", ""] + _GRAPHEMES - phonemes = ["", "", "", ""] + _PHONEMES - lr = 0.001 + batch_size: int = 256 + enc_maxlen: int = 30 * 2 + dec_maxlen: int = 40 * 2 + num_epochs: int = 50 * 2 + hidden_units: int = 64 * 8 + emb_units: int = 64 * 4 + graphemes: list[str] = ["", "", ""] + _GRAPHEMES + phonemes: list[str] = ["", "", "", ""] + _PHONEMES + lr: float = 0.001 hp = _Hparams() -def _load_vocab(): +def _load_vocab() -> tuple[dict[str, int], dict[int, str], dict[str, int], dict[int, str]]: g2idx = {g: idx for idx, g in enumerate(hp.graphemes)} idx2g = dict(enumerate(hp.graphemes)) @@ -49,12 +52,38 @@ def _load_vocab(): class Thai_W2P: - def __init__(self): + graphemes: list[str] + phonemes: list[str] + g2idx: dict[str, int] + idx2g: dict[int, str] + p2idx: dict[str, int] + idx2p: dict[int, str] + checkpoint: Optional[str] + variables: "NDArray" + enc_emb: "NDArray" + enc_w_ih: "NDArray" + enc_w_hh: "NDArray" + enc_b_ih: "NDArray" + enc_b_hh: "NDArray" + dec_emb: "NDArray" + dec_w_ih: "NDArray" + dec_w_hh: "NDArray" + dec_b_ih: "NDArray" + dec_b_hh: "NDArray" + fc_w: "NDArray" + fc_b: "NDArray" + word: str + + def __init__(self) -> None: super().__init__() - self.graphemes = hp.graphemes - self.phonemes = hp.phonemes + self.graphemes: list[str] = hp.graphemes + self.phonemes: list[str] = hp.phonemes + self.g2idx: dict[str, int] + self.idx2g: dict[int, str] + self.p2idx: dict[str, int] + self.idx2p: dict[int, str] self.g2idx, self.idx2g, self.p2idx, self.idx2p = _load_vocab() - self.checkpoint = get_corpus_path(_MODEL_NAME, version="0.2") + self.checkpoint: Optional[str] = get_corpus_path(_MODEL_NAME, version="0.2") if self.checkpoint is None: download(_MODEL_NAME, version="0.2") self.checkpoint = get_corpus_path(_MODEL_NAME) @@ -67,37 +96,37 @@ def __init__(self): def _load_variables(self) -> None: if self.checkpoint is None: raise RuntimeError("checkpoint path is not set") - self.variables = np.load(self.checkpoint, allow_pickle=True) + self.variables: "NDArray" = np.load(self.checkpoint, allow_pickle=True) # (29, 64). (len(graphemes), emb) - self.enc_emb = self.variables.item().get("encoder.emb.weight") + self.enc_emb: "NDArray" = self.variables.item().get("encoder.emb.weight") # (3*128, 64) - self.enc_w_ih = self.variables.item().get("encoder.rnn.weight_ih_l0") + self.enc_w_ih: "NDArray" = self.variables.item().get("encoder.rnn.weight_ih_l0") # (3*128, 128) - self.enc_w_hh = self.variables.item().get("encoder.rnn.weight_hh_l0") + self.enc_w_hh: "NDArray" = self.variables.item().get("encoder.rnn.weight_hh_l0") # (3*128,) - self.enc_b_ih = self.variables.item().get("encoder.rnn.bias_ih_l0") + self.enc_b_ih: "NDArray" = self.variables.item().get("encoder.rnn.bias_ih_l0") # (3*128,) - self.enc_b_hh = self.variables.item().get("encoder.rnn.bias_hh_l0") + self.enc_b_hh: "NDArray" = self.variables.item().get("encoder.rnn.bias_hh_l0") # (74, 64). (len(phonemes), emb) - self.dec_emb = self.variables.item().get("decoder.emb.weight") + self.dec_emb: "NDArray" = self.variables.item().get("decoder.emb.weight") # (3*128, 64) - self.dec_w_ih = self.variables.item().get("decoder.rnn.weight_ih_l0") + self.dec_w_ih: "NDArray" = self.variables.item().get("decoder.rnn.weight_ih_l0") # (3*128, 128) - self.dec_w_hh = self.variables.item().get("decoder.rnn.weight_hh_l0") + self.dec_w_hh: "NDArray" = self.variables.item().get("decoder.rnn.weight_hh_l0") # (3*128,) - self.dec_b_ih = self.variables.item().get("decoder.rnn.bias_ih_l0") + self.dec_b_ih: "NDArray" = self.variables.item().get("decoder.rnn.bias_ih_l0") # (3*128,) - self.dec_b_hh = self.variables.item().get("decoder.rnn.bias_hh_l0") + self.dec_b_hh: "NDArray" = self.variables.item().get("decoder.rnn.bias_hh_l0") # (74, 128) - self.fc_w = self.variables.item().get("decoder.fc.weight") + self.fc_w: "NDArray" = self.variables.item().get("decoder.fc.weight") # (74,) - self.fc_b = self.variables.item().get("decoder.fc.bias") + self.fc_b: "NDArray" = self.variables.item().get("decoder.fc.bias") - def _sigmoid(self, x): + def _sigmoid(self, x: np.ndarray) -> np.ndarray: return 1 / (1 + np.exp(-x)) - def _grucell(self, x, h, w_ih, w_hh, b_ih, b_hh): + def _grucell(self, x: np.ndarray, h: np.ndarray, w_ih: np.ndarray, w_hh: np.ndarray, b_ih: np.ndarray, b_hh: np.ndarray) -> np.ndarray: rzn_ih = np.matmul(x, w_ih.T) + b_ih rzn_hh = np.matmul(h, w_hh.T) + b_hh @@ -118,7 +147,7 @@ def _grucell(self, x, h, w_ih, w_hh, b_ih, b_hh): return h - def _gru(self, x, steps, w_ih, w_hh, b_ih, b_hh, h0=None) -> np.ndarray: + def _gru(self, x: np.ndarray, steps: int, w_ih: np.ndarray, w_hh: np.ndarray, b_ih: np.ndarray, b_hh: np.ndarray, h0: Optional[np.ndarray] = None) -> np.ndarray: if h0 is None: h0 = np.zeros((x.shape[0], w_hh.shape[1]), np.float32) h = h0 # initial hidden state @@ -128,17 +157,17 @@ def _gru(self, x, steps, w_ih, w_hh, b_ih, b_hh, h0=None) -> np.ndarray: h = self._grucell(x[:, t, :], h, w_ih, w_hh, b_ih, b_hh) # (b, h) outputs[:, t, ::] = h - return outputs # type: ignore[no-any-return] + return outputs def _encode(self, word: str) -> np.ndarray: chars = list(word) + [""] x = [self.g2idx.get(char, self.g2idx[""]) for char in chars] x = np.take(self.enc_emb, np.expand_dims(x, 0), axis=0) - return x # type: ignore[no-any-return] + return x def _short_word(self, word: str) -> Optional[str]: - self.word = word + self.word: str = word if self.word.endswith("."): self.word = self.word.replace(".", "") self.word = "-".join([i + "อ" for i in list(self.word)]) diff --git a/pythainlp/transliterate/wunsen.py b/pythainlp/transliterate/wunsen.py index 2331888bc..c069aba43 100644 --- a/pythainlp/transliterate/wunsen.py +++ b/pythainlp/transliterate/wunsen.py @@ -41,7 +41,7 @@ def transliterate( jp_input: Optional[str] = None, zh_sandhi: Optional[bool] = None, system: Optional[str] = None, - ): + ) -> str: """Use Wunsen for transliteration :param str text: text to be transliterated to Thai text. diff --git a/pythainlp/ulmfit/__init__.py b/pythainlp/ulmfit/__init__.py index e16bac311..8537550e2 100644 --- a/pythainlp/ulmfit/__init__.py +++ b/pythainlp/ulmfit/__init__.py @@ -13,7 +13,7 @@ https://arxiv.org/abs/1801.06146 """ -__all__ = [ +__all__: list[str] = [ "THWIKI_LSTM", "ThaiTokenizer", "document_vector", diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index b0615dac7..2e89c8b00 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -7,7 +7,7 @@ import collections from collections.abc import Callable, Collection -from typing import Optional +from typing import Any, Optional import numpy as np import torch @@ -172,7 +172,9 @@ def process_thai( return res # type: ignore[no-any-return] -def document_vector(text: str, learn, data, agg: str = "mean"): +def document_vector( + text: str, learn: Any, data: Any, agg: str = "mean" +) -> np.ndarray: """This function vectorizes Thai input text into a 400 dimension vector using :class:`fastai` language model and data bunch. @@ -224,10 +226,12 @@ def document_vector(text: str, learn, data, agg: str = "mean"): else: raise ValueError("Aggregate by mean or sum") - return res + return res # type: ignore[no-any-return] -def merge_wgts(em_sz, wgts, itos_pre, itos_new): +def merge_wgts( + em_sz: int, wgts: dict[str, Any], itos_pre: list[str], itos_new: list[str] +) -> dict[str, torch.Tensor]: """This function is to insert new vocab into an existing model named `wgts` and update the model's weights for new vocab with the average embedding. diff --git a/pythainlp/ulmfit/preprocess.py b/pythainlp/ulmfit/preprocess.py index 0c73dfdcf..0174453af 100644 --- a/pythainlp/ulmfit/preprocess.py +++ b/pythainlp/ulmfit/preprocess.py @@ -101,7 +101,7 @@ def replace_rep_after(text: str) -> str: 'กาxxrep7 ' """ - def _replace_rep(m): + def _replace_rep(m: re.Match[str]) -> str: c, cc = m.groups() return f"{c}{_TK_REP}{len(cc) + 1} " @@ -224,7 +224,7 @@ def replace_rep_nonum(text: str) -> str: """ - def _replace_rep(m): + def _replace_rep(m: re.Match[str]) -> str: c, _ = m.groups() return f"{c} {_TK_REP} " diff --git a/pythainlp/ulmfit/tokenizer.py b/pythainlp/ulmfit/tokenizer.py index 00810e8a8..cf98e6245 100644 --- a/pythainlp/ulmfit/tokenizer.py +++ b/pythainlp/ulmfit/tokenizer.py @@ -13,13 +13,15 @@ class BaseTokenizer: """Basic class for a tokenizer function. (codes from `fastai`)""" - def __init__(self, lang: str): - self.lang = lang + lang: str + + def __init__(self, lang: str) -> None: + self.lang: str = lang def tokenizer(self, t: str) -> list[str]: return t.split(" ") - def add_special_cases(self, toks: Collection[str]): + def add_special_cases(self, toks: Collection[str]) -> None: pass @@ -29,8 +31,10 @@ class ThaiTokenizer(BaseTokenizer): (see: https://docs.fast.ai/text.transform#BaseTokenizer) """ - def __init__(self, lang: str = "th"): - self.lang = lang + lang: str + + def __init__(self, lang: str = "th") -> None: + self.lang: str = lang @staticmethod def tokenizer(text: str) -> list[str]: @@ -64,5 +68,5 @@ def tokenizer(text: str) -> list[str]: """ return thai2fit_tokenizer().word_tokenize(text) # type: ignore[no-any-return] - def add_special_cases(self, toks): + def add_special_cases(self, toks: Collection[str]) -> None: pass diff --git a/pythainlp/util/__init__.py b/pythainlp/util/__init__.py index 068e51623..880957170 100644 --- a/pythainlp/util/__init__.py +++ b/pythainlp/util/__init__.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 """Utility functions, like date conversion and digit conversion""" -__all__ = [ +__all__: list[str] = [ "Trie", "abbreviation_to_full_text", "arabic_digit_to_thai_digit", diff --git a/pythainlp/util/date.py b/pythainlp/util/date.py index cee0521b3..0d6033143 100644 --- a/pythainlp/util/date.py +++ b/pythainlp/util/date.py @@ -116,7 +116,7 @@ } -def convert_years(year: str, src="be", target="ad") -> str: +def convert_years(year: str, src: str = "be", target: str = "ad") -> str: """Convert years :param int year: Year @@ -214,8 +214,8 @@ def thai_strptime( fmt: str, year: str = "be", add_year: Optional[int] = None, - tzinfo=ZoneInfo("Asia/Bangkok"), -): + tzinfo: Optional[ZoneInfo] = ZoneInfo("Asia/Bangkok"), +) -> datetime: """Thai strptime :param str text: text diff --git a/pythainlp/util/digitconv.py b/pythainlp/util/digitconv.py index 21cd95029..e3e842230 100644 --- a/pythainlp/util/digitconv.py +++ b/pythainlp/util/digitconv.py @@ -5,7 +5,7 @@ from __future__ import annotations -_arabic_thai = { +_arabic_thai: dict[str, str] = { "0": "๐", "1": "๑", "2": "๒", @@ -18,7 +18,7 @@ "9": "๙", } -_thai_arabic = { +_thai_arabic: dict[str, str] = { "๐": "0", "๑": "1", "๒": "2", @@ -31,7 +31,7 @@ "๙": "9", } -_digit_spell = { +_digit_spell: dict[str, str] = { "0": "ศูนย์", "1": "หนึ่ง", "2": "สอง", diff --git a/pythainlp/util/emojiconv.py b/pythainlp/util/emojiconv.py index eb3121820..8014c2c94 100644 --- a/pythainlp/util/emojiconv.py +++ b/pythainlp/util/emojiconv.py @@ -7,8 +7,9 @@ from __future__ import annotations import re +from typing import Pattern -_emoji_th = { +_emoji_th: dict[str, str] = { "😀": "หน้ายิ้มยิงฟัน", "😁": "ยิ้มยิงฟันตายิ้ม", "😂": "ร้องไห้ดีใจ", @@ -1832,7 +1833,7 @@ _delimiter = ":" -def emoji_to_thai(text: str, delimiters=(_delimiter, _delimiter)) -> str: +def emoji_to_thai(text: str, delimiters: tuple[str, str] = (_delimiter, _delimiter)) -> str: """Converts emojis to their Thai meanings. :param str text: Text with emojis diff --git a/pythainlp/util/morse.py b/pythainlp/util/morse.py index 378eb04c7..763dc5039 100644 --- a/pythainlp/util/morse.py +++ b/pythainlp/util/morse.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -THAI_MORSE_CODE = { +THAI_MORSE_CODE: dict[str, str] = { "ก": "--.", "ข": "-.-.", "ค": "-.-", @@ -74,7 +74,7 @@ "อ": "-...-", } -ENGLISH_MORSE_CODE = { +ENGLISH_MORSE_CODE: dict[str, str] = { "A": ".-", "B": "-...", "C": "-.-.", @@ -121,11 +121,11 @@ "(": "-.--.-", } -decodingeng = {} +decodingeng: dict[str, str] = {} for key, val in ENGLISH_MORSE_CODE.items(): decodingeng[val] = key -decodingthai = {} +decodingthai: dict[str, str] = {} for key, val in THAI_MORSE_CODE.items(): decodingthai[val.replace(" ", "")] = key diff --git a/pythainlp/util/normalize.py b/pythainlp/util/normalize.py index 16c527d65..6e361a37e 100644 --- a/pythainlp/util/normalize.py +++ b/pythainlp/util/normalize.py @@ -6,7 +6,7 @@ from __future__ import annotations import re -from typing import Union +from typing import Pattern, Union from pythainlp import thai_above_vowels as above_v from pythainlp import thai_below_vowels as below_v @@ -17,13 +17,13 @@ from pythainlp.tokenize import word_tokenize from pythainlp.tools import warn_deprecation -_DANGLING_CHARS = f"{above_v}{below_v}{tonemarks}\u0e3a\u0e4c\u0e4d\u0e4e" -_RE_REMOVE_DANGLINGS = re.compile(f"^[{_DANGLING_CHARS}]+") -_RE_REMOVE_DANGLINGS_AFTER_SPACE = re.compile(f" +[{_DANGLING_CHARS}]+") +_DANGLING_CHARS: str = f"{above_v}{below_v}{tonemarks}\u0e3a\u0e4c\u0e4d\u0e4e" +_RE_REMOVE_DANGLINGS: Pattern[str] = re.compile(f"^[{_DANGLING_CHARS}]+") +_RE_REMOVE_DANGLINGS_AFTER_SPACE: Pattern[str] = re.compile(f" +[{_DANGLING_CHARS}]+") -_ZERO_WIDTH_CHARS = "\u200b\u200c" # ZWSP, ZWNJ +_ZERO_WIDTH_CHARS: str = "\u200b\u200c" # ZWSP, ZWNJ -_REORDER_PAIRS = [ +_REORDER_PAIRS: list[tuple[str, str]] = [ ("\u0e40\u0e40", "\u0e41"), # Sara E + Sara E -> Sara Ae ( f"([{tonemarks}\u0e4c]+)([{above_v}{below_v}]+)", @@ -41,10 +41,10 @@ ] # VOWELS + Phinthu, Thanthakhat, Nikhahit, Yamakkan -_NOREPEAT_CHARS = ( +_NOREPEAT_CHARS: str = ( f"{follow_v}{lead_v}{above_v}{below_v}\u0e3a\u0e4c\u0e4d\u0e4e" ) -_NOREPEAT_PAIRS = list( +_NOREPEAT_PAIRS: list[tuple[str, str]] = list( zip([f"({ch}[ ]*)+{ch}" for ch in _NOREPEAT_CHARS], _NOREPEAT_CHARS) ) @@ -60,7 +60,7 @@ ) -def _last_char(matchobj): # to be used with _RE_NOREPEAT_TONEMARKS +def _last_char(matchobj: re.Match[str]) -> str: # to be used with _RE_NOREPEAT_TONEMARKS return matchobj.group(0)[-1] diff --git a/pythainlp/util/numtoword.py b/pythainlp/util/numtoword.py index 0314bed28..a69a66945 100644 --- a/pythainlp/util/numtoword.py +++ b/pythainlp/util/numtoword.py @@ -12,9 +12,9 @@ from typing import Optional -__all__ = ["bahttext", "num_to_thaiword"] +__all__: list[str] = ["bahttext", "num_to_thaiword"] -_VALUES = [ +_VALUES: list[str] = [ "", "หนึ่ง", "สอง", @@ -26,8 +26,8 @@ "แปด", "เก้า", ] -_PLACES = ["", "สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน"] -_EXCEPTIONS = {"หนึ่งสิบ": "สิบ", "สองสิบ": "ยี่สิบ", "สิบหนึ่ง": "สิบเอ็ด"} +_PLACES: list[str] = ["", "สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน"] +_EXCEPTIONS: dict[str, str] = {"หนึ่งสิบ": "สิบ", "สองสิบ": "ยี่สิบ", "สิบหนึ่ง": "สิบเอ็ด"} def bahttext(number: float) -> str: diff --git a/pythainlp/util/syllable.py b/pythainlp/util/syllable.py index 6e90cdf91..1e14b601d 100644 --- a/pythainlp/util/syllable.py +++ b/pythainlp/util/syllable.py @@ -7,9 +7,11 @@ import re +from typing import Pattern + from pythainlp import thai_consonants, thai_tonemarks -spelling_class = { +spelling_class: dict[str, list[str]] = { "กง": list("ง"), "กม": list("ม"), "เกย": list("ย"), @@ -20,39 +22,39 @@ "กบ": list("บปภพฟ"), } -thai_consonants_all = set(thai_consonants) +thai_consonants_all: set[str] = set(thai_consonants) thai_consonants_all.remove("อ") -_temp = list("".join(["".join(v) for v in spelling_class.values()])) -not_spelling_class = [j for j in thai_consonants_all if j not in _temp] +_temp: list[str] = list("".join(["".join(v) for v in spelling_class.values()])) +not_spelling_class: list[str] = [j for j in thai_consonants_all if j not in _temp] # vowel's short sound -short = "ะัิึุ" -re_short = re.compile("เ(.*)ะ|แ(.*)ะ|เ(.*)อะ|โ(.*)ะ|เ(.*)าะ", re.U) -pattern = re.compile("เ(.*)า", re.U) # เ-า is live syllable +short: str = "ะัิึุ" +re_short: Pattern[str] = re.compile("เ(.*)ะ|แ(.*)ะ|เ(.*)อะ|โ(.*)ะ|เ(.*)าะ", re.U) +pattern: Pattern[str] = re.compile("เ(.*)า", re.U) # เ-า is live syllable -_check_1 = [] +_check_1: list[str] = [] # These spelling consonant ares live syllables. for i in ["กง", "กน", "กม", "เกย", "เกอว"]: _check_1.extend(spelling_class[i]) # These spelling consonants are dead syllables. -_check_2 = spelling_class["กก"] + spelling_class["กบ"] + spelling_class["กด"] +_check_2: list[str] = spelling_class["กก"] + spelling_class["กบ"] + spelling_class["กด"] -thai_low_sonorants = list("งนมยรลว") -thai_low_aspirates = list("คชซทพฟฮ") -thai_low_irregular = list("ฆญณธภฅฌฑฒฬ") +thai_low_sonorants: list[str] = list("งนมยรลว") +thai_low_aspirates: list[str] = list("คชซทพฟฮ") +thai_low_irregular: list[str] = list("ฆญณธภฅฌฑฒฬ") -thai_mid_plains = list("กจดตบปอฎฏ") +thai_mid_plains: list[str] = list("กจดตบปอฎฏ") -thai_high_aspirates = list("ขฉถผฝสห") -thai_high_irregular = list("ศษฃฐ") -thai_initial_consonant_type = { +thai_high_aspirates: list[str] = list("ขฉถผฝสห") +thai_high_irregular: list[str] = list("ศษฃฐ") +thai_initial_consonant_type: dict[str, list[str]] = { "low": thai_low_sonorants + thai_low_aspirates + thai_low_irregular, "mid": thai_mid_plains, "high": thai_high_aspirates + thai_high_irregular, } -thai_initial_consonant_to_type = {} +thai_initial_consonant_to_type: dict[str, str] = {} for k, v in thai_initial_consonant_type.items(): for i in v: diff --git a/pythainlp/util/thai.py b/pythainlp/util/thai.py index 3707a6849..1601a623f 100644 --- a/pythainlp/util/thai.py +++ b/pythainlp/util/thai.py @@ -22,12 +22,12 @@ thai_vowels, ) -_DEFAULT_IGNORE_CHARS = string.whitespace + string.digits + string.punctuation -_TH_FIRST_CHAR_ASCII = 3584 -_TH_LAST_CHAR_ASCII = 3711 +_DEFAULT_IGNORE_CHARS: str = string.whitespace + string.digits + string.punctuation +_TH_FIRST_CHAR_ASCII: int = 3584 +_TH_LAST_CHAR_ASCII: int = 3711 # A comprehensive map of Thai characters to their descriptive names. -THAI_CHAR_NAMES = { +THAI_CHAR_NAMES: dict[str, str] = { # Consonants **{char: char for char in thai_consonants}, # Vowels and Signs diff --git a/pythainlp/util/trie.py b/pythainlp/util/trie.py index 6863291ef..193e3365a 100644 --- a/pythainlp/util/trie.py +++ b/pythainlp/util/trie.py @@ -46,14 +46,17 @@ class Trie(Iterable[str]): # output: 5 """ + words: set[str] + root: Node + class Node: __slots__ = "end", "children" - def __init__(self): - self.end = False - self.children = {} + def __init__(self) -> None: + self.end: bool = False + self.children: dict[str, Trie.Node] = {} - def __init__(self, words: Iterable[str]): + def __init__(self, words: Iterable[str]) -> None: self.words = set(words) self.root = Trie.Node() diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 7ad0dd0c8..c8dd257c7 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -11,20 +11,20 @@ import re from functools import lru_cache -from typing import Optional, Union +from typing import Optional, Pattern, Union from pythainlp.corpus import thai_words from pythainlp.tokenize import Tokenizer -_ptn_digits = r"(|หนึ่ง|เอ็ด|สอง|ยี่|สาม|สี่|ห้า|หก|เจ็ด|แปด|เก้า)" -_ptn_six_figures = ( +_ptn_digits: str = r"(|หนึ่ง|เอ็ด|สอง|ยี่|สาม|สี่|ห้า|หก|เจ็ด|แปด|เก้า)" +_ptn_six_figures: str = ( rf"({_ptn_digits}แสน)?({_ptn_digits}หมื่น)?({_ptn_digits}พัน)?" rf"({_ptn_digits}ร้อย)?({_ptn_digits}สิบ)?{_ptn_digits}?" ) -_ptn_thai_numerals = rf"(ลบ)?({_ptn_six_figures}ล้าน)*{_ptn_six_figures}" -_re_thai_numerals = re.compile(_ptn_thai_numerals) +_ptn_thai_numerals: str = rf"(ลบ)?({_ptn_six_figures}ล้าน)*{_ptn_six_figures}" +_re_thai_numerals: Pattern[str] = re.compile(_ptn_thai_numerals) -_digits = { +_digits: dict[str, int] = { # "ศูนย์" was excluded as a special case "หนึ่ง": 1, "เอ็ด": 1, @@ -38,7 +38,7 @@ "แปด": 8, "เก้า": 9, } -_powers_of_10 = { +_powers_of_10: dict[str, int] = { "สิบ": 10, "ร้อย": 100, "พัน": 1000, @@ -46,7 +46,7 @@ "แสน": 100000, # "ล้าน" was excluded as a special case } -_valid_tokens = set(_digits.keys()) | set(_powers_of_10.keys()) | {"ล้าน", "ลบ"} +_valid_tokens: set[str] = set(_digits.keys()) | set(_powers_of_10.keys()) | {"ล้าน", "ลบ"} @lru_cache diff --git a/pythainlp/wangchanberta/core.py b/pythainlp/wangchanberta/core.py index 526bb55e1..7f1ffb32a 100644 --- a/pythainlp/wangchanberta/core.py +++ b/pythainlp/wangchanberta/core.py @@ -5,7 +5,11 @@ import re import warnings -from typing import Union +from typing import TYPE_CHECKING, Union + +if TYPE_CHECKING: + from transformers import CamembertTokenizer, PreTrainedModel, PreTrainedTokenizerBase + from transformers.pipelines import TokenClassificationPipeline from pythainlp.tokenize import word_tokenize @@ -13,7 +17,7 @@ _tokenizer = None -def _get_tokenizer(): +def _get_tokenizer() -> CamembertTokenizer: """Get the tokenizer, initializing it if necessary.""" global _tokenizer if _tokenizer is None: @@ -32,9 +36,16 @@ def _get_tokenizer(): class ThaiNameTagger: + dataset_name: str + grouped_entities: bool + classify_tokens: TokenClassificationPipeline + json_ner: list[dict[str, str]] + output: str + sent_ner: list[tuple[str, str]] + def __init__( self, dataset_name: str = "thainer", grouped_entities: bool = True - ): + ) -> None: """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ @@ -46,9 +57,9 @@ def __init__( """ from transformers import pipeline - self.dataset_name = dataset_name - self.grouped_entities = grouped_entities - self.classify_tokens = pipeline( + self.dataset_name: str = dataset_name + self.grouped_entities: bool = grouped_entities + self.classify_tokens: TokenClassificationPipeline = pipeline( task="ner", tokenizer=_get_tokenizer(), model=f"airesearch/{_model_name}", @@ -57,12 +68,12 @@ def __init__( grouped_entities=self.grouped_entities, ) - def _IOB(self, tag): + def _IOB(self, tag: str) -> str: if tag != "O": return "B-" + tag return "O" - def _clear_tag(self, tag): + def _clear_tag(self, tag: str) -> str: return tag.replace("B-", "").replace("I-", "") def get_ner( @@ -87,10 +98,10 @@ def get_ner( stacklevel=2, ) text = re.sub(" ", "<_>", text) - self.json_ner = self.classify_tokens(text) - self.output = "" + self.json_ner: list[dict[str, str]] = self.classify_tokens(text) + self.output: str = "" if self.grouped_entities and self.dataset_name == "thainer": - self.sent_ner = [ + self.sent_ner: list[tuple[str, str]] = [ ( i["word"].replace("<_>", " ").replace("▁", ""), self._IOB(i["entity_group"]), @@ -144,6 +155,9 @@ def get_ner( class NamedEntityRecognition: + tokenizer: PreTrainedTokenizerBase + model: PreTrainedModel + def __init__( self, model: str = "pythainlp/thainer-corpus-v2-base-model" ) -> None: @@ -155,22 +169,22 @@ def __init__( """ from transformers import AutoModelForTokenClassification, AutoTokenizer - self.tokenizer = AutoTokenizer.from_pretrained(model) - self.model = AutoModelForTokenClassification.from_pretrained(model) + self.tokenizer: PreTrainedTokenizerBase = AutoTokenizer.from_pretrained(model) + self.model: PreTrainedModel = AutoModelForTokenClassification.from_pretrained(model) - def _fix_span_error(self, words, ner): + def _fix_span_error(self, words: list[int], ner: list[str]) -> list[tuple[str, str]]: _ner = [] _ner = ner _new_tag = [] for i, j in zip(words, _ner): - i = self.tokenizer.decode(i) - if i.isspace() and j.startswith("B-"): + i_decoded = self.tokenizer.decode(i) + if i_decoded.isspace() and j.startswith("B-"): j = "O" - if i in ("", "", ""): + if i_decoded in ("", "", ""): continue - if i == "<_>": - i = " " - _new_tag.append((i, j)) + if i_decoded == "<_>": + i_decoded = " " + _new_tag.append((i_decoded, j)) return _new_tag def get_ner( @@ -232,7 +246,7 @@ def get_ner( sent += "" return sent - return ner_tag # type: ignore[no-any-return] + return ner_tag def segment(text: str) -> list[str]: diff --git a/pythainlp/word_vector/__init__.py b/pythainlp/word_vector/__init__.py index 00c8423ae..9515f05a4 100644 --- a/pythainlp/word_vector/__init__.py +++ b/pythainlp/word_vector/__init__.py @@ -6,7 +6,7 @@ Initial code from https://github.com/cstorm125/thai2fit """ -__all__ = [ +__all__: list[str] = [ "WordVector", ] diff --git a/pythainlp/word_vector/core.py b/pythainlp/word_vector/core.py index b3fb99a38..1fd7ee4ef 100644 --- a/pythainlp/word_vector/core.py +++ b/pythainlp/word_vector/core.py @@ -43,6 +43,10 @@ def __init__(self, model_name: str = "thai2fit_wv") -> None: * *ltw2v_v1.0_15_window* - word2vec from LTW2V 1.0 and 15 window * *ltw2v_v1.0_5_window* - word2vec from LTW2V v1.0 and 5 window """ + self.model_name: str + self.model: "Word2VecKeyedVectors" + self.WV_DIM: int + self.tokenize: any # function type self.load_wordvector(model_name) def load_wordvector(self, model_name: str) -> None: @@ -52,13 +56,13 @@ def load_wordvector(self, model_name: str) -> None: """ from gensim.models import KeyedVectors - self.model_name = model_name - self.model = KeyedVectors.load_word2vec_format( + self.model_name: str = model_name + self.model: "Word2VecKeyedVectors" = KeyedVectors.load_word2vec_format( get_corpus_path(self.model_name), binary=True, unicode_errors="ignore", ) - self.WV_DIM = self.model.vector_size + self.WV_DIM: int = self.model.vector_size if self.model_name == "thai2fit_wv": self.tokenize = thai2fit_tokenizer().word_tokenize diff --git a/pythainlp/wsd/__init__.py b/pythainlp/wsd/__init__.py index 74a96bfb9..625b81963 100644 --- a/pythainlp/wsd/__init__.py +++ b/pythainlp/wsd/__init__.py @@ -3,6 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 """Thai Word Sense Disambiguation (WSD)""" -__all__ = ["get_sense"] +__all__: list[str] = ["get_sense"] from pythainlp.wsd.core import get_sense diff --git a/pythainlp/wsd/core.py b/pythainlp/wsd/core.py index 9682e4930..8ab425872 100644 --- a/pythainlp/wsd/core.py +++ b/pythainlp/wsd/core.py @@ -27,14 +27,14 @@ def __init__( self, model: str = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", device: str = "cpu", - ): + ) -> None: from sentence_transformers import SentenceTransformer - self.device = device - self.model_name = model - self.model = SentenceTransformer(self.model_name, device=self.device) + self.device: str = device + self.model_name: str = model + self.model: SentenceTransformer = SentenceTransformer(self.model_name, device=self.device) # type: ignore[assignment] - def change_device(self, device: str): + def change_device(self, device: str) -> None: from sentence_transformers import SentenceTransformer self.device = device