diff --git a/pyproject.toml b/pyproject.toml index a1c233fd9..8a079a303 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -363,3 +363,81 @@ basepython = "python" deps = ["ruff"] commands = [["ruff", "check", "pythainlp"], ["ruff", "format", "--check", "pythainlp"]] skip_install = true + +# Mypy configuration +[tool.mypy] +python_version = "3.9" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = true +disallow_untyped_decorators = false +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +strict_equality = true +show_error_codes = true +show_column_numbers = true +pretty = true + +# Per-module options for stricter checking on foundation modules +[[tool.mypy.overrides]] +module = [ + "pythainlp.tools.*", + "pythainlp.cli.*", +] +disallow_untyped_defs = true +disallow_incomplete_defs = true + +# Ignore missing imports for optional dependencies +[[tool.mypy.overrides]] +module = [ + "attacut.*", + "bpemb.*", + "budoux.*", + "deepcut.*", + "emoji.*", + "epitran.*", + "esupar.*", + "fairseq.*", + "fastai.*", + "fastcoref.*", + "gensim.*", + "huggingface_hub.*", + "icu.*", + "khamyo.*", + "khanaa.*", + "multiel.*", + "nlpo3.*", + "nltk.*", + "numpy.*", + "onnxruntime.*", + "oskut.*", + "pandas.*", + "panphon.*", + "phunspell.*", + "pycrfsuite.*", + "pyicu.*", + "sacremoses.*", + "sefr_cut.*", + "sentencepiece.*", + "sentence_transformers.*", + "spacy.*", + "spacy_thai.*", + "ssg.*", + "symspellpy.*", + "thai_nner.*", + "tltk.*", + "torch.*", + "tqdm.*", + "transformers.*", + "ufal.chu_liu_edmonds.*", + "word2word.*", + "wtpsplit.*", + "wunsen.*", + "yaml.*", +] +ignore_missing_imports = true diff --git a/pythainlp/cli/__init__.py b/pythainlp/cli/__init__.py index 2bf309c52..50c50be6d 100644 --- a/pythainlp/cli/__init__.py +++ b/pythainlp/cli/__init__.py @@ -22,7 +22,7 @@ CLI_NAME = "thainlp" -def make_usage(command: str) -> dict: +def make_usage(command: str) -> dict[str, str]: prog = f"{CLI_NAME} {command}" return {"prog": prog, "usage": f"{prog} [options]"} diff --git a/pythainlp/cli/benchmark.py b/pythainlp/cli/benchmark.py index f85f72fa4..b6da86d82 100644 --- a/pythainlp/cli/benchmark.py +++ b/pythainlp/cli/benchmark.py @@ -3,22 +3,28 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import argparse import json import os +from typing import TYPE_CHECKING from pythainlp import cli from pythainlp.tools import safe_print +if TYPE_CHECKING: + from collections.abc import Sequence + -def _read_file(path): +def _read_file(path: str) -> list[str]: with open(path, encoding="utf-8") as f: lines = (r.strip() for r in f.readlines()) return list(lines) class App: - def __init__(self, argv): + def __init__(self, argv: Sequence[str]) -> None: parser = argparse.ArgumentParser( prog="benchmark", description=( @@ -45,8 +51,8 @@ def __init__(self, argv): class WordTokenizationBenchmark: - def __init__(self, name, argv): - parser = argparse.ArgumentParser(**cli.make_usage("benchmark " + name)) + def __init__(self, name: str, argv: Sequence[str]) -> None: + parser = argparse.ArgumentParser(**cli.make_usage("benchmark " + name)) # type: ignore[arg-type] parser.add_argument( "--input-file", diff --git a/pythainlp/cli/data.py b/pythainlp/cli/data.py index 3004f4852..f20e4182e 100644 --- a/pythainlp/cli/data.py +++ b/pythainlp/cli/data.py @@ -4,14 +4,20 @@ """Command line for PyThaiNLP's dataset/corpus management. """ +from __future__ import annotations + import argparse +from typing import TYPE_CHECKING from pythainlp import corpus from pythainlp.tools import get_pythainlp_data_path +if TYPE_CHECKING: + from collections.abc import Sequence + class App: - def __init__(self, argv): + def __init__(self, argv: Sequence[str]) -> None: parser = argparse.ArgumentParser( prog="data", description="Manage dataset/corpus.", @@ -43,7 +49,7 @@ def __init__(self, argv): args = parser.parse_args(argv[2:3]) getattr(self, args.subcommand)(argv) - def get(self, argv): + def get(self, argv: Sequence[str]) -> None: parser = argparse.ArgumentParser( description="Download a dataset", usage="thainlp data get ", @@ -59,7 +65,7 @@ def get(self, argv): else: print("Not found.") - def rm(self, argv): + def rm(self, argv: Sequence[str]) -> None: parser = argparse.ArgumentParser( description="Remove a dataset", usage="thainlp data rm ", @@ -75,7 +81,7 @@ def rm(self, argv): else: print("Not found.") - def info(self, argv): + def info(self, argv: Sequence[str]) -> None: parser = argparse.ArgumentParser( description="Print information about a dataset", usage="thainlp data info ", @@ -92,14 +98,14 @@ def info(self, argv): else: print("Not found.") - def catalog(self, argv): + def catalog(self, argv: Sequence[str]) -> None: """Print dataset/corpus available for download.""" - corpus_db = corpus.get_corpus_db(corpus.corpus_db_url()) - corpus_db = corpus_db.json() - corpus_names = sorted(corpus_db.keys()) + corpus_db_response = corpus.get_corpus_db(corpus.corpus_db_url()) + corpus_db_dict: dict[str, dict[str, str]] = corpus_db_response.json() # type: ignore[union-attr] + corpus_names = sorted(corpus_db_dict.keys()) print("Dataset/corpus available for download:") for name in corpus_names: - print(f"- {name} {corpus_db[name]['latest_version']}", end="") + print(f"- {name} {corpus_db_dict[name]['latest_version']}", end="") corpus_info = corpus.get_corpus_db_detail(name) if corpus_info: print(f" (Local: {corpus_info['version']})") @@ -111,6 +117,6 @@ def catalog(self, argv): "Example: thainlp data get crfcut\n" ) - def path(self, argv): + def path(self, argv: Sequence[str]) -> None: """Print path of local dataset.""" print(get_pythainlp_data_path()) diff --git a/pythainlp/cli/misspell.py b/pythainlp/cli/misspell.py index 66b4601e4..fd582e278 100644 --- a/pythainlp/cli/misspell.py +++ b/pythainlp/cli/misspell.py @@ -2,15 +2,21 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import argparse import os import random +from typing import TYPE_CHECKING from pythainlp.tools.misspell import misspell +if TYPE_CHECKING: + from collections.abc import Sequence + class App: - def __init__(self, argv): + def __init__(self, argv: Sequence[str]) -> None: parser = argparse.ArgumentParser( prog="misspell", description="Generate misspelled texts from a given file.", diff --git a/pythainlp/cli/soundex.py b/pythainlp/cli/soundex.py index fc1ee9fd6..8640cce23 100644 --- a/pythainlp/cli/soundex.py +++ b/pythainlp/cli/soundex.py @@ -6,14 +6,20 @@ It takes input text from the command line. """ +from __future__ import annotations + import argparse +from typing import TYPE_CHECKING from pythainlp.soundex import DEFAULT_SOUNDEX_ENGINE, soundex from pythainlp.tools import safe_print +if TYPE_CHECKING: + from collections.abc import Sequence + class App: - def __init__(self, argv): + def __init__(self, argv: Sequence[str]) -> None: parser = argparse.ArgumentParser( prog="soundex", description="Convert a text to its sound-based index.", diff --git a/pythainlp/cli/tag.py b/pythainlp/cli/tag.py index f6e9c8df3..c10e1a74c 100644 --- a/pythainlp/cli/tag.py +++ b/pythainlp/cli/tag.py @@ -4,16 +4,25 @@ """Command line for PyThaiNLP's taggers. """ +from __future__ import annotations + import argparse +from typing import TYPE_CHECKING from pythainlp import cli from pythainlp.tag import pos_tag from pythainlp.tools import safe_print +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + class SubAppBase: - def __init__(self, name, argv): - parser = argparse.ArgumentParser(**cli.make_usage("tag " + name)) + separator: str + run: Callable[[list[str]], list[tuple[str, str]]] + + def __init__(self, name: str, argv: Sequence[str]) -> None: + parser = argparse.ArgumentParser(**cli.make_usage("tag " + name)) # type: ignore[arg-type] parser.add_argument( "text", type=str, @@ -39,7 +48,7 @@ def __init__(self, name, argv): class POSTaggingApp(SubAppBase): - def __init__(self, *args, **kwargs): + def __init__(self, *args: str, **kwargs: str) -> None: self.separator = "|" self.run = pos_tag @@ -47,7 +56,7 @@ def __init__(self, *args, **kwargs): class App: - def __init__(self, argv): + def __init__(self, argv: Sequence[str]) -> None: parser = argparse.ArgumentParser( prog="tag", description="Annotate a text with linguistic information", @@ -72,6 +81,6 @@ def __init__(self, argv): argv = argv[3:] if tag_type == "pos": - POSTaggingApp("Part-of-Speech tagging", argv) + POSTaggingApp("Part-of-Speech tagging", argv) # type: ignore[arg-type] else: print(f"Tag type not available: {tag_type}") diff --git a/pythainlp/cli/tokenize.py b/pythainlp/cli/tokenize.py index 723689896..1f3b1acdd 100644 --- a/pythainlp/cli/tokenize.py +++ b/pythainlp/cli/tokenize.py @@ -3,7 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 """Command line for PyThaiNLP's tokenizers.""" +from __future__ import annotations + import argparse +from typing import TYPE_CHECKING from pythainlp import cli from pythainlp.tokenize import ( @@ -16,6 +19,9 @@ ) from pythainlp.tools import safe_print +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + DEFAULT_SENT_TOKEN_SEPARATOR = "@@" # noqa: S105 DEFAULT_SUBWORD_TOKEN_SEPARATOR = "/" # noqa: S105 DEFAULT_SYLLABLE_TOKEN_SEPARATOR = "~" # noqa: S105 @@ -23,8 +29,12 @@ class SubAppBase: - def __init__(self, name, argv): - parser = argparse.ArgumentParser(**cli.make_usage("tokenize " + name)) + separator: str + algorithm: str + run: Callable[..., list[str]] + + def __init__(self, name: str, argv: Sequence[str]) -> None: + parser = argparse.ArgumentParser(**cli.make_usage("tokenize " + name)) # type: ignore[arg-type] parser.add_argument( "text", type=str, @@ -74,7 +84,7 @@ def __init__(self, name, argv): class WordTokenizationApp(SubAppBase): - def __init__(self, *args, **kwargs): + def __init__(self, *args: str, **kwargs: str) -> None: self.keep_whitespace = True self.algorithm = DEFAULT_WORD_TOKENIZE_ENGINE self.separator = DEFAULT_WORD_TOKEN_SEPARATOR @@ -83,7 +93,7 @@ def __init__(self, *args, **kwargs): class SentenceTokenizationApp(SubAppBase): - def __init__(self, *args, **kwargs): + def __init__(self, *args: str, **kwargs: str) -> None: self.keep_whitespace = True self.algorithm = DEFAULT_SENT_TOKENIZE_ENGINE self.separator = DEFAULT_SENT_TOKEN_SEPARATOR @@ -92,7 +102,7 @@ def __init__(self, *args, **kwargs): class SubwordTokenizationApp(SubAppBase): - def __init__(self, *args, **kwargs): + def __init__(self, *args: str, **kwargs: str) -> None: self.keep_whitespace = True self.algorithm = DEFAULT_SUBWORD_TOKENIZE_ENGINE self.separator = DEFAULT_SUBWORD_TOKEN_SEPARATOR @@ -101,7 +111,7 @@ def __init__(self, *args, **kwargs): class App: - def __init__(self, argv): + def __init__(self, argv: Sequence[str]) -> None: parser = argparse.ArgumentParser( prog="tokenize", description="Break a text into small units (tokens).", @@ -137,10 +147,10 @@ def __init__(self, argv): argv = argv[3:] if token_type.startswith("w"): - WordTokenizationApp("word", argv) + WordTokenizationApp("word", argv) # type: ignore[arg-type] elif token_type.startswith("su"): - SubwordTokenizationApp("subword", argv) + SubwordTokenizationApp("subword", argv) # type: ignore[arg-type] elif token_type.startswith("se"): - SentenceTokenizationApp("sent", argv) + SentenceTokenizationApp("sent", argv) # type: ignore[arg-type] else: safe_print(f"Token type not available: {token_type}") diff --git a/pythainlp/corpus/common.py b/pythainlp/corpus/common.py index 68a9dcc3e..d9f8f33c1 100644 --- a/pythainlp/corpus/common.py +++ b/pythainlp/corpus/common.py @@ -8,7 +8,10 @@ from __future__ import annotations import ast -from typing import Union +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Union __all__ = [ "countries", @@ -35,7 +38,7 @@ _THAI_COUNTRIES_FILENAME = "countries_th.txt" _THAI_THAILAND_PROVINCES: frozenset[str] = frozenset() -_THAI_THAILAND_PROVINCES_DETAILS: list[dict] = [] +_THAI_THAILAND_PROVINCES_DETAILS: list[dict[str, str]] = [] _THAI_THAILAND_PROVINCES_FILENAME = "thailand_provinces_th.csv" _THAI_SYLLABLES: frozenset[str] = frozenset() @@ -269,7 +272,7 @@ def thai_male_names() -> frozenset[str]: return _THAI_MALE_NAMES -def thai_dict() -> dict: +def thai_dict() -> dict[str, list[str]]: """Return Thai dictionary with definition from wiktionary. \n(See: `thai_dict\ `_) @@ -298,7 +301,7 @@ def thai_dict() -> dict: return _THAI_DICT -def thai_wsd_dict() -> dict: +def thai_wsd_dict() -> dict[str, Union[list[str], list[list[str]]]]: """Return Thai Word Sense Disambiguation dictionary with definition from wiktionary. \n(See: `thai_dict\ `_) @@ -319,13 +322,13 @@ def thai_wsd_dict() -> dict: use.extend(k) use = list(set(use)) if len(use) > 1: - _THAI_WSD_DICT["word"].append(i) - _THAI_WSD_DICT["meaning"].append(use) + _THAI_WSD_DICT["word"].append(i) # type: ignore[arg-type] + _THAI_WSD_DICT["meaning"].append(use) # type: ignore[arg-type] return _THAI_WSD_DICT -def thai_synonyms() -> dict: +def thai_synonyms() -> dict[str, Union[list[str], list[list[str]]]]: """Return Thai synonyms. \n(See: `thai_synonym\ `_) @@ -348,14 +351,14 @@ def thai_synonyms() -> dict: with open(path, newline="\n", encoding="utf-8") as csvfile: reader = csv.DictReader(csvfile, delimiter=",") for row in reader: - _THAI_SYNONYMS["word"].append(row["word"]) - _THAI_SYNONYMS["pos"].append(row["pos"]) - _THAI_SYNONYMS["synonym"].append(row["synonym"].split("|")) + _THAI_SYNONYMS["word"].append(row["word"]) # type: ignore[arg-type] + _THAI_SYNONYMS["pos"].append(row["pos"]) # type: ignore[arg-type] + _THAI_SYNONYMS["synonym"].append(row["synonym"].split("|")) # type: ignore[arg-type] return _THAI_SYNONYMS -def thai_synonym() -> dict: +def thai_synonym() -> dict[str, Union[list[str], list[list[str]]]]: warn_deprecation( "pythainlp.corpus.thai_synonym", "pythainlp.corpus.thai_synonyms", diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index cb50bce3d..22ca034e6 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -13,12 +13,15 @@ import zipfile from http.client import HTTPResponse from importlib.resources import files -from typing import Optional +from typing import TYPE_CHECKING from pythainlp import __version__ from pythainlp.corpus import corpus_db_path, corpus_db_url, corpus_path from pythainlp.tools import get_full_data_path +if TYPE_CHECKING: + from typing import Optional + _CHECK_MODE = os.getenv("PYTHAINLP_READ_MODE") _USER_AGENT = ( f"PyThaiNLP/{__version__} " @@ -35,7 +38,7 @@ def __init__(self, response: HTTPResponse) -> None: self.headers = response.headers self._content = response.read() - def json(self) -> dict: + def json(self) -> dict[str, str]: """Parse JSON content from response.""" try: return json.loads(self._content.decode("utf-8")) @@ -103,7 +106,7 @@ def path_pythainlp_corpus(filename: str) -> str: return os.path.join(corpus_path(), filename) -def get_corpus(filename: str, comments: bool = True) -> frozenset: +def get_corpus(filename: str, comments: bool = True) -> frozenset[str]: """Read corpus data from file and return a frozenset. Each line in the file will be a member of the set. diff --git a/pythainlp/corpus/icu.py b/pythainlp/corpus/icu.py index 400be2863..838ab2407 100644 --- a/pythainlp/corpus/icu.py +++ b/pythainlp/corpus/icu.py @@ -6,7 +6,7 @@ from __future__ import annotations -from pythainlp.corpus.common import get_corpus +from pythainlp.corpus.core import get_corpus _THAI_ICU_FILENAME = "icubrk_th.txt" diff --git a/pythainlp/corpus/th_en_translit.py b/pythainlp/corpus/th_en_translit.py index 57796de96..7699c4510 100644 --- a/pythainlp/corpus/th_en_translit.py +++ b/pythainlp/corpus/th_en_translit.py @@ -10,21 +10,22 @@ from __future__ import annotations +from collections import defaultdict +from importlib.resources import files +from typing import Union + __all__ = [ "get_transliteration_dict", "TRANSLITERATE_EN", "TRANSLITERATE_FOLLOW_RTSG", ] -from collections import defaultdict -from importlib.resources import files - _FILE_NAME = "th_en_transliteration_v1.4.tsv" TRANSLITERATE_EN = "en" TRANSLITERATE_FOLLOW_RTSG = "follow_rtsg" -def get_transliteration_dict() -> defaultdict: +def get_transliteration_dict() -> defaultdict[str, dict[str, list[Union[str, bool, None]]]]: """Get Thai to English transliteration dictionary. The returned dict is in dict[str, dict[List[str], List[Optional[bool]]]] format. @@ -39,7 +40,7 @@ def get_transliteration_dict() -> defaultdict: ) # use list, as one word can have multiple transliterations. - trans_dict: defaultdict[str, dict[str, list]] = defaultdict( + trans_dict: defaultdict[str, dict[str, list[Union[str, bool, None]]]] = defaultdict( lambda: {TRANSLITERATE_EN: [], TRANSLITERATE_FOLLOW_RTSG: []} ) try: diff --git a/pythainlp/corpus/volubilis.py b/pythainlp/corpus/volubilis.py index 14bbc0ed3..39f8c27ea 100644 --- a/pythainlp/corpus/volubilis.py +++ b/pythainlp/corpus/volubilis.py @@ -6,7 +6,7 @@ from __future__ import annotations -from pythainlp.corpus.common import get_corpus +from pythainlp.corpus.core import get_corpus _VOLUBILIS_WORDS = None _VOLUBILIS_FILENAME = "volubilis_words_th.txt" diff --git a/pythainlp/corpus/wikipedia.py b/pythainlp/corpus/wikipedia.py index 78283e716..2605286b2 100644 --- a/pythainlp/corpus/wikipedia.py +++ b/pythainlp/corpus/wikipedia.py @@ -6,7 +6,7 @@ from __future__ import annotations -from pythainlp.corpus.common import get_corpus +from pythainlp.corpus.core import get_corpus _WIKIPEDIA_TITLES = None _WIKIPEDIA_TITLES_FILENAME = "wikipedia_titles_th.txt" diff --git a/pythainlp/tools/core.py b/pythainlp/tools/core.py index 70283fc3a..53038af1d 100644 --- a/pythainlp/tools/core.py +++ b/pythainlp/tools/core.py @@ -15,7 +15,7 @@ def warn_deprecation( replacing_func: str = "", deprecated_version: str = "", removal_version: str = "", -): +) -> None: """Warn about the deprecation of a function. :param str deprecated_func: Name of the deprecated function.