diff --git a/build_tools/analysis/output/type_hint_analysis.json b/build_tools/analysis/output/type_hint_analysis.json index fed5a8f65..d57b6cb60 100644 --- a/build_tools/analysis/output/type_hint_analysis.json +++ b/build_tools/analysis/output/type_hint_analysis.json @@ -10,14 +10,14 @@ "pct_none": 0.0 }, "variables": { - "total": 1258, - "complete": 1220, - "none": 38, - "pct_complete": 96.97933227344993, - "pct_none": 3.0206677265500796, + "total": 1257, + "complete": 1183, + "none": 74, + "pct_complete": 94.11296738265712, + "pct_none": 5.88703261734288, "class_variables": 297, "instance_variables": 439, - "module_variables": 522 + "module_variables": 521 }, "type_aliases": { "total": 0 @@ -37,121 +37,121 @@ "complete": 2, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "augment": { "complete": 29, "incomplete": 0, "none": 0, - "mypy_errors": 12 + "mypy_errors": 0 }, "benchmarks": { "complete": 8, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "chat": { "complete": 4, "incomplete": 0, "none": 0, - "mypy_errors": 12 + "mypy_errors": 0 }, "classify": { "complete": 5, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "cli": { "complete": 21, "incomplete": 0, "none": 0, - "mypy_errors": 13 + "mypy_errors": 0 }, "coref": { "complete": 5, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "corpus": { "complete": 70, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "el": { "complete": 5, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "generate": { "complete": 15, "incomplete": 0, "none": 0, - "mypy_errors": 16 + "mypy_errors": 0 }, "khavee": { "complete": 9, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "lm": { "complete": 2, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "morpheme": { "complete": 2, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "parse": { "complete": 9, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "phayathaibert": { "complete": 19, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "soundex": { "complete": 27, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "spell": { "complete": 43, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "summarize": { "complete": 17, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "tag": { "complete": 73, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "tokenize": { "complete": 73, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "tokenizeicu": { "complete": 3, @@ -163,19 +163,19 @@ "complete": 9, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "translate": { "complete": 44, "incomplete": 0, "none": 0, - "mypy_errors": 18 + "mypy_errors": 0 }, "transliterate": { "complete": 75, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "transliterateicu": { "complete": 1, @@ -187,37 +187,72 @@ "complete": 25, "incomplete": 0, "none": 0, - "mypy_errors": 16 + "mypy_errors": 0 }, "util": { "complete": 109, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "wangchanberta": { "complete": 9, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 }, "word_vector": { "complete": 7, "incomplete": 0, "none": 0, - "mypy_errors": 16 + "mypy_errors": 0 }, "wsd": { "complete": 4, "incomplete": 0, "none": 0, - "mypy_errors": 11 + "mypy_errors": 0 } }, "functions_no_hints": [], "functions_incomplete_hints": [], "class_variables_no_hints": [], "instance_variables_no_hints": [ + { + "name": "pythainlp.augment.lm.fasttext.FastTextAug.model", + "scope": "public", + "parent_class": "FastTextAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/fasttext.py", + "line": 37 + }, + { + "name": "pythainlp.augment.lm.fasttext.FastTextAug.model", + "scope": "public", + "parent_class": "FastTextAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/lm/fasttext.py", + "line": 39 + }, + { + "name": "pythainlp.augment.word2vec.core.Word2VecAug.model", + "scope": "public", + "parent_class": "Word2VecAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/core.py", + "line": 32 + }, + { + "name": "pythainlp.augment.word2vec.core.Word2VecAug.model", + "scope": "public", + "parent_class": "Word2VecAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/core.py", + "line": 34 + }, + { + "name": "pythainlp.augment.word2vec.core.Word2VecAug.model", + "scope": "public", + "parent_class": "Word2VecAug", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/augment/word2vec/core.py", + "line": 38 + }, { "name": "pythainlp.chat.core.ChatBotModel.history", "scope": "public", @@ -225,26 +260,173 @@ "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/chat/core.py", "line": 22 }, + { + "name": "pythainlp.generate.core.Unigram.counts", + "scope": "public", + "parent_class": "Unigram", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/core.py", + "line": 43 + }, + { + "name": "pythainlp.generate.core.Unigram.counts", + "scope": "public", + "parent_class": "Unigram", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/core.py", + "line": 45 + }, { "name": "pythainlp.generate.core.Unigram._word_prob", "scope": "private", "parent_class": "Unigram", "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/generate/core.py", - "line": 83 + "line": 85 }, { "name": "pythainlp.tag.crfchunk.CRFchunk._model_file_ctx", "scope": "private", "parent_class": "CRFchunk", "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/crfchunk.py", - "line": 112 + "line": 114 + }, + { + "name": "pythainlp.tag.thainer.ThaiNameTagger.pos_tag_name", + "scope": "public", + "parent_class": "ThaiNameTagger", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tag/thainer.py", + "line": 128 + }, + { + "name": "pythainlp.tokenize.attacut.AttacutTokenizer._MODEL_NAME", + "scope": "private", + "parent_class": "AttacutTokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/attacut.py", + "line": 26 + }, + { + "name": "pythainlp.tokenize.core.Tokenizer.__trie_dict", + "scope": "private", + "parent_class": "Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/core.py", + "line": 944 + }, + { + "name": "pythainlp.tokenize.core.Tokenizer.__trie_dict", + "scope": "private", + "parent_class": "Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/core.py", + "line": 946 + }, + { + "name": "pythainlp.tokenize.core.Tokenizer.__engine", + "scope": "private", + "parent_class": "Tokenizer", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/core.py", + "line": 996 }, { "name": "pythainlp.tokenize.multi_cut.LatticeString.unique", "scope": "public", "parent_class": "LatticeString", "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/multi_cut.py", - "line": 45 + "line": 50 + }, + { + "name": "pythainlp.tokenize.multi_cut.LatticeString.multi", + "scope": "public", + "parent_class": "LatticeString", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/tokenize/multi_cut.py", + "line": 52 + }, + { + "name": "pythainlp.translate.core.Translate.model", + "scope": "public", + "parent_class": "Translate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/core.py", + "line": 77 + }, + { + "name": "pythainlp.translate.core.Translate.model", + "scope": "public", + "parent_class": "Translate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/core.py", + "line": 81 + }, + { + "name": "pythainlp.translate.core.Translate.model", + "scope": "public", + "parent_class": "Translate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/core.py", + "line": 85 + }, + { + "name": "pythainlp.translate.core.Translate.model", + "scope": "public", + "parent_class": "Translate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/core.py", + "line": 89 + }, + { + "name": "pythainlp.translate.core.Translate.model", + "scope": "public", + "parent_class": "Translate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/core.py", + "line": 93 + }, + { + "name": "pythainlp.translate.core.Translate.model", + "scope": "public", + "parent_class": "Translate", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/core.py", + "line": 97 + }, + { + "name": "pythainlp.translate.en_th.EnThTranslator._model", + "scope": "private", + "parent_class": "EnThTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/en_th.py", + "line": 91 + }, + { + "name": "pythainlp.translate.small100.Small100Translator.model", + "scope": "public", + "parent_class": "Small100Translator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/small100.py", + "line": 42 + }, + { + "name": "pythainlp.translate.th_fr.ThFrTranslator.model_thfr", + "scope": "public", + "parent_class": "ThFrTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/th_fr.py", + "line": 56 + }, + { + "name": "pythainlp.translate.zh_th.ThZhTranslator.model_thzh", + "scope": "public", + "parent_class": "ThZhTranslator", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/translate/zh_th.py", + "line": 50 + }, + { + "name": "pythainlp.transliterate.thai2rom.Attn.attn", + "scope": "public", + "parent_class": "Attn", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thai2rom.py", + "line": 211 + }, + { + "name": "pythainlp.transliterate.thaig2p.Attn.attn", + "scope": "public", + "parent_class": "Attn", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/thaig2p.py", + "line": 229 + }, + { + "name": "pythainlp.transliterate.w2p.Thai_W2P.checkpoint", + "scope": "public", + "parent_class": "Thai_W2P", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/w2p.py", + "line": 92 }, { "name": "pythainlp.transliterate.w2p.Thai_W2P.word", @@ -358,6 +540,41 @@ "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/wangchanberta/core.py", "line": 130 }, + { + "name": "pythainlp.word_vector.core.WordVector.model_name", + "scope": "public", + "parent_class": "WordVector", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/word_vector/core.py", + "line": 59 + }, + { + "name": "pythainlp.word_vector.core.WordVector.model", + "scope": "public", + "parent_class": "WordVector", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/word_vector/core.py", + "line": 60 + }, + { + "name": "pythainlp.word_vector.core.WordVector.WV_DIM", + "scope": "public", + "parent_class": "WordVector", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/word_vector/core.py", + "line": 65 + }, + { + "name": "pythainlp.word_vector.core.WordVector.tokenize", + "scope": "public", + "parent_class": "WordVector", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/word_vector/core.py", + "line": 68 + }, + { + "name": "pythainlp.word_vector.core.WordVector.tokenize", + "scope": "public", + "parent_class": "WordVector", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/word_vector/core.py", + "line": 70 + }, { "name": "pythainlp.wsd.core._SentenceTransformersModel.device", "scope": "public", @@ -392,6 +609,18 @@ "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/ancient/aksonhan.py", "line": 22 }, + { + "name": "pythainlp.cli.stdout", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/__init__.py", + "line": 18 + }, + { + "name": "pythainlp.cli.stderr", + "scope": "public", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/cli/__init__.py", + "line": 19 + }, { "name": "pythainlp.spell.wanchanberta_thai_grammarly.tagging_model", "scope": "public", @@ -404,6 +633,24 @@ "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/spell/wanchanberta_thai_grammarly.py", "line": 106 }, + { + "name": "pythainlp.transliterate.royin._vowel_patterns", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/royin.py", + "line": 73 + }, + { + "name": "pythainlp.transliterate.royin._vowel_patterns", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/royin.py", + "line": 74 + }, + { + "name": "pythainlp.transliterate.royin._vowel_patterns", + "scope": "private", + "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/transliterate/royin.py", + "line": 75 + }, { "name": "pythainlp.util.morse.unknown", "scope": "public", @@ -462,7 +709,7 @@ "name": "pythainlp.util.syllable.unknown", "scope": "public", "file": "/home/runner/work/pythainlp/pythainlp/pythainlp/util/syllable.py", - "line": 63 + "line": 68 }, { "name": "pythainlp.wsd.core.unknown", diff --git a/pythainlp/augment/wordnet.py b/pythainlp/augment/wordnet.py index 496811aeb..01b2aa706 100644 --- a/pythainlp/augment/wordnet.py +++ b/pythainlp/augment/wordnet.py @@ -12,9 +12,10 @@ import itertools from collections import OrderedDict -from typing import Any, Callable, Optional +from typing import Callable, Optional from nltk.corpus import wordnet as wn +from nltk.corpus.reader.wordnet import Synset from pythainlp.corpus import wordnet from pythainlp.tag import pos_tag @@ -121,7 +122,7 @@ class WordNetAug: synonyms: list[str] list_synsets: list p2w_pos: Optional[str] - synset: Any + synset: Synset syn: str synonyms_without_duplicates: list[str] list_words: list[str] diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index 2ba01ef4c..1a0a3d484 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -5,7 +5,7 @@ import re import sys -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Union if TYPE_CHECKING: import numpy as np @@ -42,7 +42,7 @@ def _f1(precision: float, recall: float) -> float: return 2 * precision * recall / (precision + recall) -def _flatten_result(my_dict: dict, sep: str = ":") -> dict[str, Any]: +def _flatten_result(my_dict: dict, sep: str = ":") -> dict[str, Union[int, str]]: """Flatten two-dimension dictionary. Use keys in the first dimension as a prefix for keys in the second dimension. @@ -56,7 +56,7 @@ def _flatten_result(my_dict: dict, sep: str = ":") -> dict[str, Any]: :param str sep: separator between the two keys (default: ":") :return: a one-dimension dictionary with keys combined - :rtype: dict[str, Any] + :rtype: dict[str, Union[int, str]] """ return { f"{k1}{sep}{k2}": v @@ -133,7 +133,7 @@ def preprocessing(txt: str, remove_space: bool = True) -> str: return txt -def compute_stats(ref_sample: str, raw_sample: str) -> dict[str, Any]: +def compute_stats(ref_sample: str, raw_sample: str) -> dict[str, dict[str, Union[int, str]]]: """Compute statistics for tokenization quality These statistics include: @@ -150,7 +150,7 @@ def compute_stats(ref_sample: str, raw_sample: str) -> dict[str, Any]: :param str samples: samples that we want to evaluate :return: metrics at character- and word-level and indicators of correctly tokenized words - :rtype: dict[str, Any] + :rtype: dict[str, dict[str, Union[int, str]]] """ import numpy as np @@ -166,11 +166,11 @@ def compute_stats(ref_sample: str, raw_sample: str) -> dict[str, Any]: c_pos_pred = c_pos_pred[c_pos_pred < ref_sample_arr.shape[0]] c_neg_pred = c_neg_pred[c_neg_pred < ref_sample_arr.shape[0]] - c_tp: np.intp = np.sum(ref_sample_arr[c_pos_pred] == 1) - c_fp: np.intp = np.sum(ref_sample_arr[c_pos_pred] == 0) + c_tp: int = int(np.sum(ref_sample_arr[c_pos_pred] == 1)) + c_fp: int = int(np.sum(ref_sample_arr[c_pos_pred] == 0)) - c_tn: np.intp = np.sum(ref_sample_arr[c_neg_pred] == 0) - c_fn: np.intp = np.sum(ref_sample_arr[c_neg_pred] == 1) + c_tn: int = int(np.sum(ref_sample_arr[c_neg_pred] == 0)) + c_fn: int = int(np.sum(ref_sample_arr[c_neg_pred] == 1)) # Compute word-level statistics @@ -183,7 +183,7 @@ def compute_stats(ref_sample: str, raw_sample: str) -> dict[str, Any]: word_boundaries, ss_boundaries ) - correctly_tokenised_words: np.intp = np.sum(tokenization_indicators) + correctly_tokenised_words: int = int(np.sum(tokenization_indicators)) tokenization_indicators_str = list(map(str, tokenization_indicators)) @@ -196,8 +196,8 @@ def compute_stats(ref_sample: str, raw_sample: str) -> dict[str, Any]: }, "word_level": { "correctly_tokenised_words": correctly_tokenised_words, - "total_words_in_sample": np.sum(sample_arr), - "total_words_in_ref_sample": np.sum(ref_sample_arr), + "total_words_in_sample": int(np.sum(sample_arr)), + "total_words_in_ref_sample": int(np.sum(ref_sample_arr)), }, "global": { "tokenisation_indicators": "".join(tokenization_indicators_str) diff --git a/pythainlp/chat/core.py b/pythainlp/chat/core.py index 584a06d95..a5a699d92 100644 --- a/pythainlp/chat/core.py +++ b/pythainlp/chat/core.py @@ -3,19 +3,21 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, cast +from typing import TYPE_CHECKING, Optional, cast if TYPE_CHECKING: import torch + from pythainlp.generate.wangchanglm import WangChanGLM + class ChatBotModel: history: list[tuple[str, str]] - model: Any + model: "WangChanGLM" def __init__(self) -> None: """Chat using AI generation""" - self.history: list[tuple[str, str]] = [] + self.history = [] def reset_chat(self) -> None: """Reset chat by cleaning history""" @@ -49,7 +51,7 @@ def load_model( if model_name == "wangchanglm": from pythainlp.generate.wangchanglm import WangChanGLM - self.model: Any = WangChanGLM() + self.model = WangChanGLM() self.model.load_model( model_path="pythainlp/wangchanglm-7.5B-sft-en-sharded", return_dict=return_dict, diff --git a/pythainlp/classify/param_free.py b/pythainlp/classify/param_free.py index fdc09d493..e94b65ef0 100644 --- a/pythainlp/classify/param_free.py +++ b/pythainlp/classify/param_free.py @@ -35,8 +35,8 @@ def __init__( if model_path: self.load(model_path) else: - self.training_data: "NDArray[Any]" = np.array(training_data) - self.cx2_list: list[int] = self.train() + self.training_data = np.array(training_data) + self.cx2_list = self.train() def train(self) -> list[int]: temp_list = [] @@ -112,7 +112,5 @@ def load(self, path: str) -> None: with open(path, "r", encoding="utf-8") as f: data = json.load(f) - self.cx2_list: list[int] = data["cx2_list"] - self.training_data: "NDArray[Any]" = np.array( - data["training_data"] - ) + self.cx2_list = data["cx2_list"] + self.training_data = np.array(data["training_data"]) diff --git a/pythainlp/cli/tag.py b/pythainlp/cli/tag.py index cc2e69685..c41ba8a3f 100644 --- a/pythainlp/cli/tag.py +++ b/pythainlp/cli/tag.py @@ -6,7 +6,7 @@ from __future__ import annotations import argparse -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from pythainlp import cli from pythainlp.tag import pos_tag @@ -38,7 +38,7 @@ def __init__(self, name: str, argv: Sequence[str]) -> None: ) args = parser.parse_args(argv) - self.args: Any = args + self.args: argparse.Namespace = args tokens = args.text.split(args.separator) result = self.run(tokens) diff --git a/pythainlp/cli/tokenize.py b/pythainlp/cli/tokenize.py index 06c6e151a..214f68f0e 100644 --- a/pythainlp/cli/tokenize.py +++ b/pythainlp/cli/tokenize.py @@ -74,7 +74,7 @@ def __init__(self, name: str, argv: Sequence[str]) -> None: parser.set_defaults(keep_whitespace=True) args = parser.parse_args(argv) - self.args: Any = args + self.args: argparse.Namespace = args cli.exit_if_empty(args.text, parser) result = self.run( diff --git a/pythainlp/coref/_fastcoref.py b/pythainlp/coref/_fastcoref.py index 79c041ac5..f1db6bc47 100644 --- a/pythainlp/coref/_fastcoref.py +++ b/pythainlp/coref/_fastcoref.py @@ -3,13 +3,21 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Optional, TypedDict if TYPE_CHECKING: - from fastcoref.modeling import CorefModel + from fastcoref.modeling import CorefModel, CorefResult from spacy.language import Language +class CorefResultDict(TypedDict): + """Dictionary representation of coreference resolution results.""" + + text: str + clusters_string: list[list[str]] + clusters: list[list[tuple[int, int]]] + + class FastCoref: def __init__( self, @@ -34,14 +42,14 @@ def __init__( self.model_name, device=device, nlp=self.nlp ) - def _to_json(self, _predict: Any) -> dict[str, Any]: + def _to_json(self, _predict: "CorefResult") -> CorefResultDict: return { "text": _predict.text, "clusters_string": _predict.get_clusters(as_strings=True), "clusters": _predict.get_clusters(as_strings=False), } - def predict(self, texts: list[str]) -> list[dict]: + def predict(self, texts: list[str]) -> list[CorefResultDict]: return [ self._to_json(pred) for pred in self.model.predict(texts=texts) ] diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 740f03b53..d774652c3 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -38,9 +38,9 @@ class _ResponseWrapper: _content: bytes def __init__(self, response: HTTPResponse) -> None: - self.status_code: int = response.status - self.headers: HTTPMessage = response.headers - self._content: bytes = response.read() + self.status_code = response.status + self.headers = response.headers + self._content = response.read() def json(self) -> dict[str, Any]: """Parse JSON content from response.""" diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index f328c94b5..aab4d6ac5 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -16,7 +16,10 @@ from typing import TYPE_CHECKING, Any, Union if TYPE_CHECKING: + from pathlib import Path + import pandas as pd + from fastai.basic_train import Learner from fastai.text import ( AWD_LSTM, LMDataBunch, @@ -48,7 +51,7 @@ ) # get dummy data -imdb: Any = untar_data(URLs.IMDB_SAMPLE) +imdb: "Path" = untar_data(URLs.IMDB_SAMPLE) dummy_df: "pd.DataFrame" = pd.read_csv(imdb / "texts.csv") # get vocab @@ -113,7 +116,7 @@ "beta": 1, } -learn: Any = language_model_learner( +learn: "Learner" = language_model_learner( data_lm, AWD_LSTM, config=config, pretrained=False, **trn_args ) diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index e27ad3eec..b8fff4d4a 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -7,10 +7,11 @@ import re import warnings from collections.abc import Callable -from typing import TYPE_CHECKING, Any, Union +from typing import TYPE_CHECKING, Union if TYPE_CHECKING: from transformers import CamembertTokenizer + from transformers.pipelines.base import Pipeline from transformers import ( CamembertTokenizer, @@ -217,7 +218,7 @@ def __init__(self) -> None: self.model_for_masked_lm: AutoModelForMaskedLM = ( AutoModelForMaskedLM.from_pretrained(_model_name) ) - self.model: Any = pipeline( # transformers.Pipeline + self.model: "Pipeline" = pipeline( "fill-mask", tokenizer=self.tokenizer, model=self.model_for_masked_lm, diff --git a/pythainlp/spell/words_spelling_correction.py b/pythainlp/spell/words_spelling_correction.py index 76c1d9641..1be5a6e16 100644 --- a/pythainlp/spell/words_spelling_correction.py +++ b/pythainlp/spell/words_spelling_correction.py @@ -67,24 +67,18 @@ def __init__( ) except Exception as e: raise RuntimeError(f"An unexpected error occurred: {e}") from e - self.model_dir: str = model_dir - self.nn_model_path: str = nn_model_path - self.bucket: int = bucket - self.nb_words: int = nb_words - self.minn: int = minn - self.maxn: int = maxn + self.model_dir = model_dir + self.nn_model_path = nn_model_path + self.bucket = bucket + self.nb_words = nb_words + self.minn = minn + self.maxn = maxn # Load data and models - self.vocabulary: list[str] - self.embeddings: "NDArray[np.float32]" self.vocabulary, self.embeddings = self._load_embeddings() - self.words_for_suggestion: "NDArray[np.str_]" = ( - self._load_suggestion_words(words_list) - ) - self.nn_session: "InferenceSession" = self._load_onnx_session( - nn_model_path - ) - self.embedding_dim: int = self.embeddings.shape[1] + self.words_for_suggestion = self._load_suggestion_words(words_list) + self.nn_session = self._load_onnx_session(nn_model_path) + self.embedding_dim = self.embeddings.shape[1] def _load_embeddings(self) -> tuple[list[str], NDArray[np.float32]]: """Loads embeddings matrix and vocabulary list.""" @@ -272,9 +266,9 @@ class Words_Spelling_Correction(FastTextEncoder): list_word: list[str] def __init__(self) -> None: - self.model_name: str = "pythainlp/word-spelling-correction-char2vec" - self.model_path: str = get_hf_hub(self.model_name) - self.model_onnx: str = get_hf_hub( + self.model_name = "pythainlp/word-spelling-correction-char2vec" + self.model_path = get_hf_hub(self.model_name) + self.model_onnx = get_hf_hub( self.model_name, "nearest_neighbors.onnx" ) with open( @@ -282,7 +276,7 @@ def __init__(self) -> None: self.model_name, "list_word-spelling-correction-char2vec.txt" ) ) as f: - self.list_word: list[str] = list(map(str.strip, f.readlines())) + self.list_word = list(map(str.strip, f.readlines())) super().__init__(self.model_path, self.model_onnx, self.list_word) diff --git a/pythainlp/summarize/keybert.py b/pythainlp/summarize/keybert.py index 676664bff..08c8218d3 100644 --- a/pythainlp/summarize/keybert.py +++ b/pythainlp/summarize/keybert.py @@ -14,24 +14,25 @@ from collections import Counter from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Optional, Union from pythainlp.corpus import thai_stopwords from pythainlp.tokenize import word_tokenize if TYPE_CHECKING: import numpy as np + from transformers.pipelines.base import Pipeline class KeyBERT: - ft_pipeline: Any + ft_pipeline: "Pipeline" def __init__( self, model_name: str = "airesearch/wangchanberta-base-att-spm-uncased" ) -> None: from transformers import pipeline - self.ft_pipeline: Any = pipeline( + self.ft_pipeline: "Pipeline" = pipeline( "feature-extraction", tokenizer=model_name, model=model_name, diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py index efe425827..d3c2ead37 100644 --- a/pythainlp/tag/crfchunk.py +++ b/pythainlp/tag/crfchunk.py @@ -6,7 +6,7 @@ import types from contextlib import AbstractContextManager from importlib.resources import as_file, files -from typing import Any, Optional +from typing import Any, Optional, Union from pycrfsuite import Tagger as CRFTagger @@ -17,12 +17,12 @@ def _is_stopword(word: str) -> bool: # check Thai stopword return word in thai_stopwords() -def _doc2features(tokens: list[tuple[str, str]], index: int) -> dict[str, Any]: +def _doc2features(tokens: list[tuple[str, str]], index: int) -> dict[str, Union[str, bool]]: """`tokens` = a POS-tagged sentence [(w1, t1), ...] `index` = the index of the token we want to extract features for """ word, pos = tokens[index] - f: dict[str, Any] = { + f: dict[str, Union[str, bool]] = { "word": word, "word_is_stopword": _is_stopword(word), "pos": pos, @@ -55,7 +55,7 @@ def _doc2features(tokens: list[tuple[str, str]], index: int) -> dict[str, Any]: return f -def extract_features(doc: list[tuple[str, str]]) -> list[dict[str, Any]]: +def extract_features(doc: list[tuple[str, str]]) -> list[dict[str, Union[str, bool]]]: return [_doc2features(doc, i) for i in range(0, len(doc))] @@ -75,26 +75,24 @@ class CRFchunk: corpus: str _model_file_ctx: Optional[AbstractContextManager[Any]] tagger: CRFTagger - xseq: list[dict[str, Any]] + xseq: list[dict[str, Union[str, bool]]] def __init__(self, corpus: str = "orchidpp") -> None: - self.corpus: str = corpus - self._model_file_ctx: Optional[AbstractContextManager[Any]] = None + self.corpus = corpus + self._model_file_ctx = None self.load_model(self.corpus) def load_model(self, corpus: str) -> None: - self.tagger: CRFTagger = CRFTagger() + self.tagger = CRFTagger() if corpus == "orchidpp": corpus_files = files("pythainlp.corpus") model_file = corpus_files.joinpath("crfchunk_orchidpp.model") - self._model_file_ctx: Optional[AbstractContextManager[Any]] = ( - as_file(model_file) - ) + self._model_file_ctx = as_file(model_file) model_path = self._model_file_ctx.__enter__() self.tagger.open(str(model_path)) def parse(self, token_pos: list[tuple[str, str]]) -> list[str]: - self.xseq: list[dict[str, Any]] = extract_features(token_pos) + self.xseq = extract_features(token_pos) return self.tagger.tag(self.xseq) # type: ignore[no-any-return] def __enter__(self) -> CRFchunk: diff --git a/pythainlp/tag/named_entity.py b/pythainlp/tag/named_entity.py index e55ad0dcd..4f519e0bf 100644 --- a/pythainlp/tag/named_entity.py +++ b/pythainlp/tag/named_entity.py @@ -5,7 +5,31 @@ from __future__ import annotations -from typing import Any, Union +from typing import TYPE_CHECKING, Any, Union + +if TYPE_CHECKING: + from types import ModuleType + + from pythainlp.phayathaibert.core import NamedEntityTagger + from pythainlp.tag.thai_nner import ThaiNNER + from pythainlp.tag.thainer import ThaiNameTagger + from pythainlp.wangchanberta.core import ( + NamedEntityRecognition, + ) + from pythainlp.wangchanberta.core import ( + ThaiNameTagger as WangchanbertaThaiNameTagger, + ) + +# Type alias for NER engine types +NEREngineType = Union[ + "ThaiNNER", + "ModuleType", + "ThaiNameTagger", + "NamedEntityRecognition", + "WangchanbertaThaiNameTagger", + "NamedEntityTagger", + None, +] class NER: @@ -31,38 +55,36 @@ class NER: """ name_engine: str - engine: Any + engine: NEREngineType def __init__( self, engine: str = "thainer-v2", corpus: str = "thainer" ) -> None: - self.name_engine: str - self.engine: Any self.load_engine(engine=engine, corpus=corpus) def load_engine(self, engine: str, corpus: str) -> None: - self.name_engine: str = engine - self.engine: Any = None + self.name_engine = engine + self.engine = None # Engines that ignore corpus parameter if engine == "thai-nner": from pythainlp.tag.thai_nner import ThaiNNER - self.engine: Any = ThaiNNER() + self.engine = ThaiNNER() elif engine == "tltk": from pythainlp.tag import tltk - self.engine: Any = tltk + self.engine = tltk # Corpus-specific engines elif corpus == "thainer": if engine == "thainer": from pythainlp.tag.thainer import ThaiNameTagger - self.engine: Any = ThaiNameTagger() + self.engine = ThaiNameTagger() elif engine == "thainer-v2": from pythainlp.wangchanberta import NamedEntityRecognition - self.engine: Any = NamedEntityRecognition( + self.engine = NamedEntityRecognition( model="pythainlp/thainer-corpus-v2-base-model" ) elif engine == "wangchanberta": @@ -70,14 +92,14 @@ def load_engine(self, engine: str, corpus: str) -> None: ThaiNameTagger as WangchanbertaThaiNameTagger, ) # noqa: I001,E501 - self.engine: Any = WangchanbertaThaiNameTagger( + self.engine = WangchanbertaThaiNameTagger( dataset_name=corpus ) elif corpus == "thainer-v2": if engine == "phayathaibert": from pythainlp.phayathaibert.core import NamedEntityTagger - self.engine: Any = NamedEntityTagger() + self.engine = NamedEntityTagger() if self.engine is None: raise ValueError( @@ -130,16 +152,15 @@ class NNER: * *thai_nner* - Thai NER engine """ - engine: Any + engine: "ThaiNNER" def __init__(self, engine: str = "thai_nner") -> None: - self.engine: Any self.load_engine(engine) def load_engine(self, engine: str = "thai_nner") -> None: from pythainlp.tag.thai_nner import ThaiNNER - self.engine: Any = ThaiNNER() + self.engine = ThaiNNER() def tag( self, text: str, top_level_only: bool = False diff --git a/pythainlp/tag/unigram.py b/pythainlp/tag/unigram.py index 02e3e2ed1..b6ed243a2 100644 --- a/pythainlp/tag/unigram.py +++ b/pythainlp/tag/unigram.py @@ -7,7 +7,7 @@ import json import os -from typing import Any, Optional +from typing import Optional from pythainlp.corpus import corpus_path, get_corpus_path from pythainlp.tag import blackboard, orchid @@ -26,14 +26,14 @@ _TUD_FILENAME: str = "pos_tud_unigram.json" _TUD_PATH: str = os.path.join(corpus_path(), _TUD_FILENAME) -_ORCHID_TAGGER: Optional[dict[str, Any]] = None -_PUD_TAGGER: Optional[dict[str, Any]] = None -_BLACKBOARD_TAGGER: Optional[dict[str, Any]] = None -_TDTB_TAGGER: Optional[dict[str, Any]] = None -_TUD_TAGGER: Optional[dict[str, Any]] = None +_ORCHID_TAGGER: Optional[dict[str, str]] = None +_PUD_TAGGER: Optional[dict[str, str]] = None +_BLACKBOARD_TAGGER: Optional[dict[str, str]] = None +_TDTB_TAGGER: Optional[dict[str, str]] = None +_TUD_TAGGER: Optional[dict[str, str]] = None -def _orchid_tagger() -> dict[str, Any]: +def _orchid_tagger() -> dict[str, str]: global _ORCHID_TAGGER if not _ORCHID_TAGGER: with open(_ORCHID_PATH, encoding="utf-8-sig") as fh: @@ -41,7 +41,7 @@ def _orchid_tagger() -> dict[str, Any]: return _ORCHID_TAGGER -def _pud_tagger() -> dict[str, Any]: +def _pud_tagger() -> dict[str, str]: global _PUD_TAGGER if not _PUD_TAGGER: with open(_PUD_PATH, encoding="utf-8-sig") as fh: @@ -49,7 +49,7 @@ def _pud_tagger() -> dict[str, Any]: return _PUD_TAGGER -def _blackboard_tagger() -> dict[str, Any]: +def _blackboard_tagger() -> dict[str, str]: global _BLACKBOARD_TAGGER if not _BLACKBOARD_TAGGER: path = get_corpus_path(_BLACKBOARD_NAME) @@ -60,7 +60,7 @@ def _blackboard_tagger() -> dict[str, Any]: return _BLACKBOARD_TAGGER -def _thai_tdtb() -> dict[str, Any]: +def _thai_tdtb() -> dict[str, str]: global _TDTB_TAGGER if not _TDTB_TAGGER: with open(_TDTB_PATH, encoding="utf-8-sig") as fh: @@ -68,7 +68,7 @@ def _thai_tdtb() -> dict[str, Any]: return _TDTB_TAGGER -def _tud_tagger() -> dict[str, Any]: +def _tud_tagger() -> dict[str, str]: global _TUD_TAGGER if not _TUD_TAGGER: with open(_TUD_PATH, encoding="utf-8-sig") as fh: @@ -77,7 +77,7 @@ def _tud_tagger() -> dict[str, Any]: def _find_tag( - words: list[str], dictdata: dict[str, Any], default_tag: str = "" + words: list[str], dictdata: dict[str, str], default_tag: str = "" ) -> list[tuple[str, str]]: keys = list(dictdata.keys()) return [ diff --git a/pythainlp/tag/wangchanberta_onnx.py b/pythainlp/tag/wangchanberta_onnx.py index fb96e49ad..26bf710a3 100644 --- a/pythainlp/tag/wangchanberta_onnx.py +++ b/pythainlp/tag/wangchanberta_onnx.py @@ -41,13 +41,13 @@ def __init__( SessionOptions, ) - self.model_name: str = model_name - self.model_version: str = model_version - self.options: SessionOptions = SessionOptions() + self.model_name = model_name + self.model_version = model_version + self.options = SessionOptions() self.options.graph_optimization_level = ( GraphOptimizationLevel.ORT_ENABLE_ALL ) - self.session: InferenceSession = InferenceSession( + self.session = InferenceSession( get_path_folder_corpus( self.model_name, self.model_version, file_onnx ), @@ -55,8 +55,8 @@ def __init__( providers=providers, ) self.session.disable_fallback() - self.outputs_name: str = self.session.get_outputs()[0].name - self.sp: spm.SentencePieceProcessor = spm.SentencePieceProcessor( + self.outputs_name = self.session.get_outputs()[0].name + self.sp = spm.SentencePieceProcessor( model_file=get_path_folder_corpus( self.model_name, self.model_version, "sentencepiece.bpe.model" ) @@ -67,8 +67,8 @@ def __init__( ), encoding="utf-8-sig", ) as fh: - self._json: dict[str, Any] = json.load(fh) - self.id2tag: dict[str, str] = self._json["id2label"] + self._json = json.load(fh) + self.id2tag = self._json["id2label"] def build_tokenizer(self, sent: str) -> dict[str, "np.ndarray"]: import numpy as np diff --git a/pythainlp/transliterate/ipa.py b/pythainlp/transliterate/ipa.py index a4e8d026b..bdffe8b08 100644 --- a/pythainlp/transliterate/ipa.py +++ b/pythainlp/transliterate/ipa.py @@ -11,11 +11,9 @@ from __future__ import annotations -from typing import Any - import epitran -_EPI_THA: Any = epitran.Epitran("tha-Thai") +_EPI_THA: epitran.Epitran = epitran.Epitran("tha-Thai") def transliterate(text: str) -> str: diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index 166e8c6e1..2715cc3c0 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -41,31 +41,31 @@ def __init__(self) -> None: Now supports Thai to Latin (romanization) """ # get the model, download it if it's not available locally - self.__model_filename: str = get_corpus_path(_MODEL_NAME) # type: ignore[assignment] + self.__model_filename = get_corpus_path(_MODEL_NAME) # type: ignore[assignment] loader = torch.load(self.__model_filename, map_location=device) INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT = loader["encoder_params"] OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT = loader["decoder_params"] - self._maxlength: int = 100 + self._maxlength = 100 - self._char_to_ix: "Dict[str, int]" = loader["char_to_ix"] - self._ix_to_char: "Dict[int, str]" = loader["ix_to_char"] - self._target_char_to_ix: "Dict[str, int]" = loader["target_char_to_ix"] - self._ix_to_target_char: "Dict[int, str]" = loader["ix_to_target_char"] + self._char_to_ix = loader["char_to_ix"] + self._ix_to_char = loader["ix_to_char"] + self._target_char_to_ix = loader["target_char_to_ix"] + self._ix_to_target_char = loader["ix_to_target_char"] # encoder/ decoder # Restore the model and construct the encoder and decoder. - self._encoder: "Encoder" = Encoder( + self._encoder = Encoder( INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT ) - self._decoder: "AttentionDecoder" = AttentionDecoder( + self._decoder = AttentionDecoder( OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT ) - self._network: "Seq2Seq" = Seq2Seq( + self._network = Seq2Seq( self._encoder, self._decoder, self._target_char_to_ix[""], @@ -130,18 +130,18 @@ def __init__( ) -> None: """Constructor""" super().__init__() - self.hidden_size: int = hidden_size - self.character_embedding: nn.Embedding = nn.Embedding( + self.hidden_size = hidden_size + self.character_embedding = nn.Embedding( vocabulary_size, embedding_size ) - self.rnn: nn.LSTM = nn.LSTM( + self.rnn = nn.LSTM( input_size=embedding_size, hidden_size=hidden_size // 2, bidirectional=True, batch_first=True, ) - self.dropout: nn.Dropout = nn.Dropout(dropout) + self.dropout = nn.Dropout(dropout) def forward( self, sequences: torch.Tensor, sequences_lengths: torch.Tensor @@ -201,15 +201,15 @@ class Attn(nn.Module): def __init__(self, method: str, hidden_size: int) -> None: super().__init__() - self.method: str = method - self.hidden_size: int = hidden_size + self.method = method + self.hidden_size = hidden_size if self.method == "general": - self.attn: nn.Linear = nn.Linear(self.hidden_size, hidden_size) + self.attn = nn.Linear(self.hidden_size, hidden_size) elif self.method == "concat": self.attn = nn.Linear(self.hidden_size * 2, hidden_size) - self.other: nn.Parameter = nn.Parameter( + self.other = nn.Parameter( torch.FloatTensor(1, hidden_size) ) @@ -268,22 +268,22 @@ def __init__( ) -> None: """Constructor""" super().__init__() - self.vocabulary_size: int = vocabulary_size - self.hidden_size: int = hidden_size - self.character_embedding: nn.Embedding = nn.Embedding( + self.vocabulary_size = vocabulary_size + self.hidden_size = hidden_size + self.character_embedding = nn.Embedding( vocabulary_size, embedding_size ) - self.rnn: nn.LSTM = nn.LSTM( + self.rnn = nn.LSTM( input_size=embedding_size + self.hidden_size, hidden_size=hidden_size, bidirectional=False, batch_first=True, ) - self.attn: Attn = Attn(method="general", hidden_size=self.hidden_size) - self.linear: nn.Linear = nn.Linear(hidden_size, vocabulary_size) + self.attn = Attn(method="general", hidden_size=self.hidden_size) + self.linear = nn.Linear(hidden_size, vocabulary_size) - self.dropout: nn.Dropout = nn.Dropout(dropout) + self.dropout = nn.Dropout(dropout) def forward( self, diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index 52ee03f99..4761e3c1b 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -13,6 +13,8 @@ if TYPE_CHECKING: import numpy as np + from fastai.basic_data import DataBunch + from fastai.basic_train import Learner from pythainlp.corpus import get_corpus_path from pythainlp.tokenize import thai2fit_tokenizer @@ -186,7 +188,7 @@ def process_thai( def document_vector( - text: str, learn: Any, data: Any, agg: str = "mean" + text: str, learn: "Learner", data: "DataBunch", agg: str = "mean" ) -> "np.ndarray": """This function vectorizes Thai input text into a 400 dimension vector using :class:`fastai` language model and data bunch. diff --git a/pythainlp/wangchanberta/core.py b/pythainlp/wangchanberta/core.py index d55be2942..f5b028b3c 100644 --- a/pythainlp/wangchanberta/core.py +++ b/pythainlp/wangchanberta/core.py @@ -61,9 +61,9 @@ def __init__( """ from transformers import pipeline - self.dataset_name: str = dataset_name - self.grouped_entities: bool = grouped_entities - self.classify_tokens: TokenClassificationPipeline = pipeline( + self.dataset_name = dataset_name + self.grouped_entities = grouped_entities + self.classify_tokens = pipeline( task="ner", tokenizer=_get_tokenizer(), model=f"airesearch/{_model_name}", diff --git a/pythainlp/word_vector/core.py b/pythainlp/word_vector/core.py index c380bb08e..d292f4751 100644 --- a/pythainlp/word_vector/core.py +++ b/pythainlp/word_vector/core.py @@ -3,7 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from typing import TYPE_CHECKING, Any +from collections.abc import Callable +from typing import TYPE_CHECKING from pythainlp.corpus import get_corpus_path from pythainlp.tokenize import thai2fit_tokenizer, word_tokenize @@ -46,7 +47,7 @@ def __init__(self, model_name: str = "thai2fit_wv") -> None: self.model_name: str self.model: "Word2VecKeyedVectors" self.WV_DIM: int - self.tokenize: Any # function type + self.tokenize: Callable[[str], list[str]] self.load_wordvector(model_name) def load_wordvector(self, model_name: str) -> None: