diff --git a/CHANGELOG.md b/CHANGELOG.md index f0d66e9a4..e2023e0b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,45 @@ and this project adheres to ## [Unreleased] +### Added + +- Type checking: add `pyrightconfig.json` to configure pyright for the project. + +### Changed + +- Type checking: add `extra_checks`, `strict_bytes`, `strict_equality_for_none`, + and `strict_optional` options to mypy configuration in `pyproject.toml`. +- `tokenize.tcc`, `tokenize.tcc_p`, `util.pronounce`: use `list()` constructor + to produce `list[str]` instead of assigning `list[LiteralString]` to `list[str]`. +- `tag._tag_perceptron.PerceptronTagger`: remove duplicate class-level type + declarations; fix incorrect re-annotations in `load()` method; use indexed + access instead of sequence unpacking for `START` and `END` lists. +- `tag.unigram`: use `cast` for lazy-loaded global tagger dicts to satisfy + pyright's return-type narrowing requirements. +- `tokenize.budoux`: add an explicit `None`-guard after lazy init to narrow + the parser type before calling `.parse()`. +- `tokenize.core`: pass `list(custom_dict)` directly to `deepcut_segment` + instead of re-annotating the parameter variable. +- `tokenize.nercut`: cast `NER.tag()` result to `list[tuple[str, str]]` to + resolve the broad return-type union. +- `summarize.core`: use `cast` instead of a bare `# type: ignore` comment. +- `translate.tokenization_small100`: remove redundant instance-level type + re-annotations for `prefix_tokens` and `cur_lang_id`; suppress false-positive + `reportRedeclaration` on property getter/setter pair. +- `transliterate.thai2rom`, `transliterate.thaig2p`: add `else: raise ValueError` + branch to attention-energy if/elif chains so `attn_energies` is always bound. +- `util.thai_lunar_date`: initialize `th_m` before the loop; add `else: raise + ValueError` for unexpected `last_day` values. +- `spell.pn`: use `cast` when converting a `dict` argument to an item list. +- `spell.symspellpy`: suppress pyright `reportReturnType` false positive caused + by `SymSpell` being unresolvable when `symspellpy` is not installed. + ### Fixed +- thai2rom, thaig2p: raise `ValueError` for unsupported attention method name, + eliminating the possibly-unbound variable warning. +- thai_lunar_date: raise `ValueError` for unexpected `last_day` values, making + the `days_in_month` variable always bound. - thai2rom_onnx: fix ONNX encoder model and fix inference bugs (#1349) - wordnet: fix AttributeError (#1354) diff --git a/pyproject.toml b/pyproject.toml index f77e4c9a1..81afc3058 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -424,13 +424,16 @@ check_untyped_defs = true disallow_untyped_defs = false disallow_incomplete_defs = false disallow_untyped_decorators = false +extra_checks = true pretty = true python_version = "3.9" show_column_numbers = true show_error_code_links = true show_error_context = true -strict_optional = true +strict_bytes = true strict_equality = true +strict_equality_for_none = true +strict_optional = true warn_no_return = true warn_redundant_casts = true warn_return_any = true diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 000000000..430e3a7fd --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,12 @@ +{ + "pythonVersion": "3.9", + "typeCheckingMode": "basic", + "reportMissingImports": "none", + "reportMissingModuleSource": "none", + "reportAttributeAccessIssue": "none", + "reportCallIssue": "none", + "reportGeneralTypeIssues": "none", + "reportInvalidTypeForm": "none", + "reportOperatorIssue": "none", + "reportOptionalMemberAccess": "none" +} diff --git a/pythainlp/spell/pn.py b/pythainlp/spell/pn.py index ac3210d2e..448464329 100644 --- a/pythainlp/spell/pn.py +++ b/pythainlp/spell/pn.py @@ -89,7 +89,9 @@ def _convert_custom_dict( ) -> list[tuple[str, int]]: """Converts a custom dictionary to a list of (str, int) tuples""" if isinstance(custom_dict, dict): - custom_dict = list(custom_dict.items()) + custom_dict = cast( # pyright: ignore[reportAssignmentType] + Iterable[tuple[str, int]], list(custom_dict.items()) + ) i = iter(custom_dict) first_member = next(i) diff --git a/pythainlp/spell/symspellpy.py b/pythainlp/spell/symspellpy.py index c592147d3..f8550aa42 100644 --- a/pythainlp/spell/symspellpy.py +++ b/pythainlp/spell/symspellpy.py @@ -80,7 +80,7 @@ def _get_sym_spell() -> SymSpell: separator="\t", encoding="utf-8-sig", ) - return _sym_spell + return _sym_spell # pyright: ignore[reportReturnType] def spell(text: str, max_edit_distance: int = 2) -> list[str]: diff --git a/pythainlp/summarize/core.py b/pythainlp/summarize/core.py index 77c9ad56d..a9510f30a 100644 --- a/pythainlp/summarize/core.py +++ b/pythainlp/summarize/core.py @@ -114,7 +114,7 @@ def summarize( sents = mT5Summarizer(model_size=size).summarize(text) else: # if engine not found, return first n sentences # sent_tokenize with str input returns list[str] - sents = sent_tokenize(text, engine="whitespace+newline")[:n] # type: ignore[assignment] + sents = cast(list[str], sent_tokenize(text, engine="whitespace+newline")[:n]) return sents diff --git a/pythainlp/tag/_tag_perceptron.py b/pythainlp/tag/_tag_perceptron.py index e9b97c410..c3590873e 100644 --- a/pythainlp/tag/_tag_perceptron.py +++ b/pythainlp/tag/_tag_perceptron.py @@ -127,13 +127,9 @@ class PerceptronTagger: END: list[str] = ["-END-", "-END2-"] AP_MODEL_LOC: str = "" - model: "AveragedPerceptron" - tagdict: dict[str, str] - classes: set[str] - def __init__(self, path: str = "") -> None: """:param str path: model path""" - self.model: "AveragedPerceptron" = AveragedPerceptron() + self.model: AveragedPerceptron = AveragedPerceptron() self.tagdict: dict[str, str] = {} self.classes: set[str] = set() if path != "": @@ -142,7 +138,8 @@ def __init__(self, path: str = "") -> None: def tag(self, tokens: Iterable[str]) -> list[tuple[str, str]]: """Tags a string `tokens`.""" - prev, prev2 = self.START + prev = self.START[0] + prev2 = self.START[1] output = [] context = self.START + [self._normalize(w) for w in tokens] + self.END @@ -181,7 +178,8 @@ def train( for sentence in sentences_list: words, tags = zip(*sentence) - prev, prev2 = self.START + prev = self.START[0] + prev2 = self.START[1] context = ( self.START + [self._normalize(w) for w in words] + self.END ) @@ -220,9 +218,9 @@ def load(self, loc: str) -> None: msg = "Missing trontagger.json file." raise OSError(msg) from ex self.model.weights = w_td_c["weights"] - self.tagdict: dict[str, list[str]] = w_td_c["tagdict"] - self.classes: list[str] = w_td_c["classes"] - self.model.classes = set(self.classes) + self.tagdict = w_td_c["tagdict"] + self.classes = set(w_td_c["classes"]) + self.model.classes = self.classes def _normalize(self, word: str) -> str: """Normalization used in pre-processing. diff --git a/pythainlp/tag/unigram.py b/pythainlp/tag/unigram.py index 88504167f..b8a606669 100644 --- a/pythainlp/tag/unigram.py +++ b/pythainlp/tag/unigram.py @@ -7,7 +7,7 @@ import json import os -from typing import Optional +from typing import Optional, cast from pythainlp.corpus import corpus_path, get_corpus_path from pythainlp.tag import blackboard, orchid @@ -38,7 +38,7 @@ def _orchid_tagger() -> dict[str, str]: if not _ORCHID_TAGGER: with open(_ORCHID_PATH, encoding="utf-8-sig") as fh: _ORCHID_TAGGER = json.load(fh) - return _ORCHID_TAGGER + return cast(dict[str, str], _ORCHID_TAGGER) def _pud_tagger() -> dict[str, str]: @@ -46,7 +46,7 @@ def _pud_tagger() -> dict[str, str]: if not _PUD_TAGGER: with open(_PUD_PATH, encoding="utf-8-sig") as fh: _PUD_TAGGER = json.load(fh) - return _PUD_TAGGER + return cast(dict[str, str], _PUD_TAGGER) def _blackboard_tagger() -> dict[str, str]: @@ -62,7 +62,7 @@ def _blackboard_tagger() -> dict[str, str]: ) with open(path, encoding="utf-8-sig") as fh: _BLACKBOARD_TAGGER = json.load(fh) - return _BLACKBOARD_TAGGER + return cast(dict[str, str], _BLACKBOARD_TAGGER) def _thai_tdtb() -> dict[str, str]: @@ -70,7 +70,7 @@ def _thai_tdtb() -> dict[str, str]: if not _TDTB_TAGGER: with open(_TDTB_PATH, encoding="utf-8-sig") as fh: _TDTB_TAGGER = json.load(fh) - return _TDTB_TAGGER + return cast(dict[str, str], _TDTB_TAGGER) def _tud_tagger() -> dict[str, str]: @@ -78,7 +78,7 @@ def _tud_tagger() -> dict[str, str]: if not _TUD_TAGGER: with open(_TUD_PATH, encoding="utf-8-sig") as fh: _TUD_TAGGER = json.load(fh) - return _TUD_TAGGER + return cast(dict[str, str], _TUD_TAGGER) def _find_tag( diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index ff133c68b..02a84f1a7 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -275,8 +275,7 @@ def word_tokenize( from pythainlp.tokenize.deepcut import segment as deepcut_segment # noqa: I001 if custom_dict: - custom_dict = list(custom_dict) # type: ignore[assignment] - segments = deepcut_segment(text, custom_dict) + segments = deepcut_segment(text, list(custom_dict)) else: segments = deepcut_segment(text) elif engine == "icu": diff --git a/pythainlp/tokenize/nercut.py b/pythainlp/tokenize/nercut.py index 9b14d5992..3ff9ad4a5 100644 --- a/pythainlp/tokenize/nercut.py +++ b/pythainlp/tokenize/nercut.py @@ -12,7 +12,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast if TYPE_CHECKING: from collections.abc import Iterable @@ -46,7 +46,7 @@ def segment( if not text: return [] - tagged_words = tagger.tag(text, pos=False) + tagged_words = cast(list[tuple[str, str]], tagger.tag(text, pos=False)) words = [] combining_word = "" diff --git a/pythainlp/tokenize/tcc.py b/pythainlp/tokenize/tcc.py index cbda6131b..dde6fe1f7 100644 --- a/pythainlp/tokenize/tcc.py +++ b/pythainlp/tokenize/tcc.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: from collections.abc import Iterator -_RE_TCC: list[str] = ( +_RE_TCC: list[str] = list( """\ c[ั]([่-๋]c)? c[ั]([่-๋]c)?k diff --git a/pythainlp/tokenize/tcc_p.py b/pythainlp/tokenize/tcc_p.py index d29f2f291..696d0ad5b 100644 --- a/pythainlp/tokenize/tcc_p.py +++ b/pythainlp/tokenize/tcc_p.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: from collections.abc import Iterator -_RE_TCC: list[str] = ( +_RE_TCC: list[str] = list( """\ เc็ck เcctาะk diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index 6625cf9a5..923814b43 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -234,12 +234,12 @@ def vocab_size(self) -> int: ) @property - def tgt_lang(self) -> str: + def tgt_lang(self) -> str: # pyright: ignore[reportRedeclaration] return self._tgt_lang @tgt_lang.setter - def tgt_lang(self, new_tgt_lang: str) -> None: - self._tgt_lang: str = new_tgt_lang + def tgt_lang(self, new_tgt_lang: str) -> None: # pyright: ignore[reportRedeclaration] + self._tgt_lang = new_tgt_lang self.set_lang_special_tokens(self._tgt_lang) def _tokenize(self, text: str) -> list[str]: @@ -434,16 +434,16 @@ def _switch_to_input_mode(self) -> None: self.set_lang_special_tokens(self.tgt_lang) def _switch_to_target_mode(self) -> None: - self.prefix_tokens: Optional[list[int]] = None - self.suffix_tokens: list[int] = [self.eos_token_id] + self.prefix_tokens = None + self.suffix_tokens = [self.eos_token_id] def set_lang_special_tokens(self, src_lang: str) -> None: """Reset the special tokens to the tgt lang setting. No prefix and suffix=[eos, tgt_lang_code].""" lang_token = self.get_lang_token(src_lang) - self.cur_lang_id: int = self.lang_token_to_id[lang_token] - self.prefix_tokens: list[int] = [self.cur_lang_id] - self.suffix_tokens: list[int] = [self.eos_token_id] + self.cur_lang_id = self.lang_token_to_id[lang_token] + self.prefix_tokens = [self.cur_lang_id] + self.suffix_tokens = [self.eos_token_id] def get_lang_token(self, lang: str) -> str: return self.lang_code_to_token[lang] diff --git a/pythainlp/util/pronounce.py b/pythainlp/util/pronounce.py index d3605674d..2a69908ba 100644 --- a/pythainlp/util/pronounce.py +++ b/pythainlp/util/pronounce.py @@ -50,7 +50,7 @@ def rhyme(word: str) -> list[str]: "โอ,เอาะ,ออ,เออะ,เออ,อำ,ใอ,ไอ,เอา,ฤ,ฤๅ,ฦ,ฦๅ", ) ) -thai_vowel: list[str] = _vowel_str.split(",") +thai_vowel: list[str] = list(_vowel_str.split(",")) thai_vowel_all: list[tuple[str, str]] = [ ("([ก-ฮ])ะ", "\\1อะ"), ("([ก-ฮ])า", "\\1อา"),