From 6b2fb216d09dd2eb63eb6334df0396101d0e30e3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Feb 2026 11:13:18 +0000 Subject: [PATCH 1/7] Initial plan From b59d023719e7da375a4801d412913dbf970cd87e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Feb 2026 11:21:04 +0000 Subject: [PATCH 2/7] Fix type hints in translate module and related files - Fixed translate/core.py to properly type the model Union - Fixed zh_th.py, th_fr.py, small100.py to return str properly - Fixed tokenization_small100.py prefix_tokens to Optional[list[int]] - Fixed en_th.py, word2word_translate.py with type ignore comments - Fixed transliterate files (umt5_thaig2p, thaig2p_v2, w2p) - Reduced mypy errors from 84 to 67 Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/translate/core.py | 19 ++++++++++++++--- pythainlp/translate/en_th.py | 6 +++--- pythainlp/translate/small100.py | 9 +++++--- pythainlp/translate/th_fr.py | 5 +++-- pythainlp/translate/tokenization_small100.py | 22 ++++++++++---------- pythainlp/translate/word2word_translate.py | 2 +- pythainlp/translate/zh_th.py | 10 +++++---- pythainlp/transliterate/thaig2p_v2.py | 2 +- pythainlp/transliterate/umt5_thaig2p.py | 2 +- pythainlp/transliterate/w2p.py | 8 +++---- 10 files changed, 52 insertions(+), 33 deletions(-) diff --git a/pythainlp/translate/core.py b/pythainlp/translate/core.py index e03708f52..545f9ff89 100644 --- a/pythainlp/translate/core.py +++ b/pythainlp/translate/core.py @@ -3,7 +3,13 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from typing import Optional +from typing import TYPE_CHECKING, Optional, Union + +if TYPE_CHECKING: + from pythainlp.translate.en_th import EnThTranslator, ThEnTranslator + from pythainlp.translate.small100 import Small100Translator + from pythainlp.translate.th_fr import ThFrTranslator + from pythainlp.translate.zh_th import ThZhTranslator, ZhThTranslator class Translate: @@ -46,14 +52,21 @@ def __init__( th2en.translate("ฉันรักแมว") # output: I love cat. """ - self.model = None + self.model: Union[ + Small100Translator, + ThEnTranslator, + EnThTranslator, + ThZhTranslator, + ZhThTranslator, + ThFrTranslator, + ] self.engine = engine self.src_lang = src_lang self.use_gpu = use_gpu self.target_lang = target_lang self.load_model() - def load_model(self): + def load_model(self) -> None: src_lang = self.src_lang target_lang = self.target_lang use_gpu = self.use_gpu diff --git a/pythainlp/translate/en_th.py b/pythainlp/translate/en_th.py index 0ec11defc..b68ada170 100644 --- a/pythainlp/translate/en_th.py +++ b/pythainlp/translate/en_th.py @@ -41,7 +41,7 @@ def _get_translate_path(model: str, *path: str) -> str: corpus_path = get_corpus_path(model, version="1.0") if corpus_path is None: return "" - return os.path.join(corpus_path, *path) # type: ignore[arg-type] + return os.path.join(corpus_path, *path) def _download_install(name: str) -> None: @@ -109,7 +109,7 @@ def translate(self, text: str) -> str: """ tokens = " ".join(self._tokenizer.tokenize(text)) translated = self._model.translate(tokens) - return translated.replace(" ", "").replace("▁", " ").strip() + return translated.replace(" ", "").replace("▁", " ").strip() # type: ignore[no-any-return] class ThEnTranslator: @@ -168,4 +168,4 @@ def translate(self, text: str) -> str: # output: I love cat. """ - return self._model.translate(text) + return self._model.translate(text) # type: ignore[no-any-return] diff --git a/pythainlp/translate/small100.py b/pythainlp/translate/small100.py index b774b9a50..d7360b52a 100644 --- a/pythainlp/translate/small100.py +++ b/pythainlp/translate/small100.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from typing import Optional + from transformers import M2M100ForConditionalGeneration from .tokenization_small100 import SMALL100Tokenizer @@ -25,7 +27,7 @@ def __init__( self.model = M2M100ForConditionalGeneration.from_pretrained( self.pretrained ) - self.tgt_lang = None + self.tgt_lang: Optional[str] = None if use_gpu: self.model = self.model.cuda() @@ -66,6 +68,7 @@ def translate(self, text: str, tgt_lang: str = "en") -> str: self.translated = self.model.generate( **self.tokenizer(text, return_tensors="pt") ) - return self.tokenizer.batch_decode( + decoded_list = self.tokenizer.batch_decode( self.translated, skip_special_tokens=True - )[0] + ) + return decoded_list[0] diff --git a/pythainlp/translate/th_fr.py b/pythainlp/translate/th_fr.py index a408c7896..3e59c332e 100644 --- a/pythainlp/translate/th_fr.py +++ b/pythainlp/translate/th_fr.py @@ -63,7 +63,8 @@ def translate(self, text: str) -> str: self.translated = self.model_thzh.generate( **self.tokenizer_thzh(text, return_tensors="pt", padding=True) ) - return [ + decoded_list = [ self.tokenizer_thzh.decode(t, skip_special_tokens=True) for t in self.translated - ][0] + ] + return decoded_list[0] diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index f14109ea0..00cfdbaf6 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -119,7 +119,7 @@ class SMALL100Tokenizer(PreTrainedTokenizer): pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP model_input_names = ["input_ids", "attention_mask"] - prefix_tokens: list[int] = [] + prefix_tokens: Optional[list[int]] = [] suffix_tokens: list[int] = [] def __init__( @@ -196,7 +196,7 @@ def __init__( @property def vocab_size(self) -> int: - return len(self.encoder) + len(self.lang_token_to_id) + self.num_madeup_words + return len(self.encoder) + len(self.lang_token_to_id) + self.num_madeup_words # type: ignore[no-any-return] @property def tgt_lang(self) -> str: @@ -208,7 +208,7 @@ def tgt_lang(self, new_tgt_lang: str) -> None: self.set_lang_special_tokens(self._tgt_lang) def _tokenize(self, text: str) -> list[str]: - return self.sp_model.encode(text, out_type=str) + return self.sp_model.encode(text, out_type=str) # type: ignore[no-any-return] def _convert_token_to_id(self, token: str) -> int: if token in self.lang_token_to_id: @@ -218,15 +218,15 @@ def _convert_token_to_id(self, token: str) -> int: def _convert_id_to_token(self, index: int) -> str: """Converts an index (integer) in a token (str) using the decoder.""" if index in self.id_to_lang_token: - return self.id_to_lang_token[index] + return self.id_to_lang_token[index] # type: ignore[no-any-return] token = self.decoder.get(index, self.unk_token) if token is None: - return self.unk_token - return token + return self.unk_token # type: ignore[no-any-return] + return token # type: ignore[no-any-return] def convert_tokens_to_string(self, tokens: list[str]) -> str: """Converts a sequence of tokens (strings for sub-words) in a single string.""" - return self.sp_model.decode(tokens) + return self.sp_model.decode(tokens) # type: ignore[no-any-return] def get_special_tokens_mask( self, @@ -257,9 +257,9 @@ def get_special_tokens_mask( token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True, - ) + ) # type: ignore[no-any-return] - prefix_ones = [1] * len(self.prefix_tokens) + prefix_ones = [1] * len(self.prefix_tokens) if self.prefix_tokens else [] suffix_ones = [1] * len(self.suffix_tokens) if token_ids_1 is None: return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones @@ -406,9 +406,9 @@ def load_spm( return spm -def load_json(path: str) -> Union[dict, list]: +def load_json(path: str) -> Union[dict[Any, Any], list[Any]]: with open(path) as f: - return json.load(f) + return json.load(f) # type: ignore[no-any-return] def save_json(data, path: str) -> None: diff --git a/pythainlp/translate/word2word_translate.py b/pythainlp/translate/word2word_translate.py index 6dea1e8cd..2193fd6c4 100644 --- a/pythainlp/translate/word2word_translate.py +++ b/pythainlp/translate/word2word_translate.py @@ -87,4 +87,4 @@ def translate(word: str, src: str, target: str) -> Optional[list[str]]: elif src == target: return [word] _engine = Word2word(src, target) - return _engine(word) + return _engine(word) # type: ignore[no-any-return] diff --git a/pythainlp/translate/zh_th.py b/pythainlp/translate/zh_th.py index 38b5a5f9b..5c5df3406 100644 --- a/pythainlp/translate/zh_th.py +++ b/pythainlp/translate/zh_th.py @@ -57,10 +57,11 @@ def translate(self, text: str) -> str: self.translated = self.model_thzh.generate( **self.tokenizer_thzh(text, return_tensors="pt", padding=True) ) - return [ + decoded_list = [ self.tokenizer_thzh.decode(t, skip_special_tokens=True) for t in self.translated - ][0] + ] + return decoded_list[0] class ZhThTranslator: @@ -108,7 +109,8 @@ def translate(self, text: str) -> str: self.translated = self.model_zhth.generate( **self.tokenizer_zhth(text, return_tensors="pt", padding=True) ) - return [ + decoded_list = [ self.tokenizer_zhth.decode(t, skip_special_tokens=True) for t in self.translated - ][0] + ] + return decoded_list[0] diff --git a/pythainlp/transliterate/thaig2p_v2.py b/pythainlp/transliterate/thaig2p_v2.py index 6ad4fa381..7300b959d 100644 --- a/pythainlp/transliterate/thaig2p_v2.py +++ b/pythainlp/transliterate/thaig2p_v2.py @@ -32,7 +32,7 @@ def __init__(self, device: str = "cpu"): ) def g2p(self, text: str) -> str: - return self.pipe(text)[0]["generated_text"] + return self.pipe(text)[0]["generated_text"] # type: ignore[no-any-return] _THAI_G2P = None diff --git a/pythainlp/transliterate/umt5_thaig2p.py b/pythainlp/transliterate/umt5_thaig2p.py index 57e9c7749..e3831d60b 100644 --- a/pythainlp/transliterate/umt5_thaig2p.py +++ b/pythainlp/transliterate/umt5_thaig2p.py @@ -32,7 +32,7 @@ def __init__(self, device: str = "cpu"): ) def g2p(self, text: str) -> str: - return self.pipe(text)[0]["generated_text"] + return self.pipe(text)[0]["generated_text"] # type: ignore[no-any-return] _THAI_G2P = None diff --git a/pythainlp/transliterate/w2p.py b/pythainlp/transliterate/w2p.py index d437403b3..30d031559 100644 --- a/pythainlp/transliterate/w2p.py +++ b/pythainlp/transliterate/w2p.py @@ -60,8 +60,8 @@ def __init__(self): self.checkpoint = get_corpus_path(_MODEL_NAME) self._load_variables() - def _load_variables(self): - self.variables = np.load(self.checkpoint, allow_pickle=True) + def _load_variables(self) -> None: + self.variables = np.load(self.checkpoint, allow_pickle=True) # type: ignore[arg-type] # (29, 64). (len(graphemes), emb) self.enc_emb = self.variables.item().get("encoder.emb.weight") # (3*128, 64) @@ -124,12 +124,12 @@ def _gru(self, x, steps, w_ih, w_hh, b_ih, b_hh, h0=None) -> np.ndarray: return outputs - def _encode(self, word: str) -> np.ndarray: + def _encode(self, word: str) -> np.ndarray: # type: ignore[type-arg] chars = list(word) + [""] x = [self.g2idx.get(char, self.g2idx[""]) for char in chars] x = np.take(self.enc_emb, np.expand_dims(x, 0), axis=0) - return x + return x # type: ignore[no-any-return] def _short_word(self, word: str) -> Optional[str]: self.word = word From 61e3e39fe95fdd9ccab36510c08c47f8e26a5c56 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Feb 2026 11:25:10 +0000 Subject: [PATCH 3/7] Fix type hints in summarize module - Fixed summarize/core.py - added cast for sent_tokenize, fixed stop_words type narrowing - Fixed summarize/freq.py - added cast for sent_tokenize - Fixed summarize/keybert.py - added type ignore for numpy operations - Added return type annotation for rank_by_frequency - Reduced mypy errors from 67 to 56 Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/summarize/core.py | 31 +++++++++++++++++++------------ pythainlp/summarize/freq.py | 4 +++- pythainlp/summarize/keybert.py | 12 ++++++------ 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/pythainlp/summarize/core.py b/pythainlp/summarize/core.py index f1c4f975d..f58204d53 100644 --- a/pythainlp/summarize/core.py +++ b/pythainlp/summarize/core.py @@ -7,7 +7,7 @@ from __future__ import annotations from collections.abc import Iterable -from typing import Optional +from typing import Optional, cast from pythainlp.summarize import ( CPE_KMUTT_THAI_SENTENCE_SUM, @@ -112,7 +112,8 @@ def summarize( sents = mT5Summarizer(model_size=size).summarize(text) else: # if engine not found, return first n sentences - sents = sent_tokenize(text, engine="whitespace+newline")[:n] + # sent_tokenize with str input returns list[str] + sents = sent_tokenize(text, engine="whitespace+newline")[:n] # type: ignore[assignment] return sents @@ -197,7 +198,7 @@ def rank_by_frequency( min_df: int = 5, tokenizer: str = "newmm", stop_words: Optional[Iterable[str]] = None, - ): + ) -> list[str]: from pythainlp.tokenize import word_tokenize from pythainlp.util.keywords import rank @@ -205,11 +206,14 @@ def rank_by_frequency( use_custom_stop_words = stop_words is not None - if use_custom_stop_words: + if use_custom_stop_words and stop_words is not None: tokens = [token for token in tokens if token not in stop_words] word_rank = rank(tokens, exclude_stopwords=not use_custom_stop_words) + if word_rank is None: + return [] + keywords = [ kw for kw, cnt in word_rank.most_common(max_keywords) @@ -223,14 +227,17 @@ def rank_by_frequency( if engine == "keybert": from .keybert import KeyBERT - keywords = KeyBERT().extract_keywords( - text, - keyphrase_ngram_range=keyphrase_ngram_range, - max_keywords=max_keywords, - min_df=min_df, - tokenizer=tokenizer, - return_similarity=False, - stop_words=stop_words, + keywords = cast( + list[str], + KeyBERT().extract_keywords( + text, + keyphrase_ngram_range=keyphrase_ngram_range, + max_keywords=max_keywords, + min_df=min_df, + tokenizer=tokenizer, + return_similarity=False, + stop_words=stop_words, + ), ) elif engine == "frequency": return rank_by_frequency( diff --git a/pythainlp/summarize/freq.py b/pythainlp/summarize/freq.py index a0bbab712..615c9982d 100644 --- a/pythainlp/summarize/freq.py +++ b/pythainlp/summarize/freq.py @@ -9,6 +9,7 @@ from collections import defaultdict from heapq import nlargest from string import punctuation +from typing import cast from pythainlp.corpus import thai_stopwords from pythainlp.tokenize import sent_tokenize, word_tokenize @@ -49,7 +50,8 @@ def __compute_frequencies( def summarize( self, text: str, n: int, tokenizer: str = "newmm" ) -> list[str]: - sents = sent_tokenize(text, engine="whitespace+newline") + # sent_tokenize with str input returns list[str] + sents = cast(list[str], sent_tokenize(text, engine="whitespace+newline")) word_tokenized_sents = [ word_tokenize(sent, engine=tokenizer) for sent in sents ] diff --git a/pythainlp/summarize/keybert.py b/pythainlp/summarize/keybert.py index 2a0dbe998..cec7f6fc1 100644 --- a/pythainlp/summarize/keybert.py +++ b/pythainlp/summarize/keybert.py @@ -152,7 +152,7 @@ def embed(self, docs: Union[str, list[str]]) -> np.ndarray: [np.array(emb[0]).mean(axis=0) for emb in embs] ) - return emb_mean + return emb_mean # type: ignore[no-any-return] def _generate_ngrams( @@ -172,7 +172,7 @@ def _generate_ngrams( f"current value={keyphrase_ngram_range}." ) - def _join_ngram(ngrams: list[tuple[str, ...]]) -> list[str]: # type: ignore[type-arg] + def _join_ngram(ngrams: list[tuple[str, ...]]) -> list[str]: ngrams_joined = [] for ng in ngrams: joined = "".join(ng) @@ -190,7 +190,7 @@ def _join_ngram(ngrams: list[tuple[str, ...]]) -> list[str]: # type: ignore[typ ngrams = [word for word in words if word.strip()] else: ngrams_tuple = zip(*[words[i:] for i in range(n)]) - ngrams = _join_ngram(list(ngrams_tuple)) # type: ignore[arg-type] + ngrams = _join_ngram(list(ngrams_tuple)) ngrams_cnt = Counter(ngrams) ngrams = [ @@ -220,10 +220,10 @@ def l2_norm(v: np.ndarray) -> np.ndarray: assert np.isclose(np.linalg.norm(result, axis=1), 1).all(), ( "Cannot normalize a vector to unit vector." ) - return result + return result # type: ignore[no-any-return] - def cosine_sim(a: np.ndarray, b: np.ndarray) -> np.ndarray: - return (np.matmul(a, b.T).T).sum(axis=1) + def cosine_sim(a: np.ndarray, b: np.ndarray) -> np.ndarray: # type: ignore[type-arg] + return (np.matmul(a, b.T).T).sum(axis=1) # type: ignore[no-any-return] doc_vector = l2_norm(doc_vector) word_vectors = l2_norm(word_vectors) From 01285427be85ae2b32e436638225333a285ec32f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Feb 2026 11:27:24 +0000 Subject: [PATCH 4/7] Fix type hints in translate, parse, and coref modules - Fixed type annotations for decoded_list in translate files (zh_th, th_fr, small100) - Removed unused type ignore comments in tokenization_small100.py - Fixed parse modules to use Optional[str] for model parameters - Fixed coref/core.py to accept Union[str, list[str]] for texts parameter - Reduced mypy errors from 56 to 46 Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/coref/core.py | 4 +++- pythainlp/parse/esupar_engine.py | 2 +- pythainlp/parse/transformers_ud.py | 4 ++-- pythainlp/parse/ud_goeswith.py | 4 ++-- pythainlp/translate/small100.py | 2 +- pythainlp/translate/th_fr.py | 2 +- pythainlp/translate/tokenization_small100.py | 6 +++--- pythainlp/translate/zh_th.py | 4 ++-- 8 files changed, 15 insertions(+), 13 deletions(-) diff --git a/pythainlp/coref/core.py b/pythainlp/coref/core.py index d0a6301c6..77799f526 100644 --- a/pythainlp/coref/core.py +++ b/pythainlp/coref/core.py @@ -3,11 +3,13 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from typing import Union + _MODEL = None def coreference_resolution( - texts: list[str], model_name: str = "han-coref-v1.0", device: str = "cpu" + texts: Union[str, list[str]], model_name: str = "han-coref-v1.0", device: str = "cpu" ) -> list[dict]: """Coreference Resolution diff --git a/pythainlp/parse/esupar_engine.py b/pythainlp/parse/esupar_engine.py index 28d493cd2..33b59f14a 100644 --- a/pythainlp/parse/esupar_engine.py +++ b/pythainlp/parse/esupar_engine.py @@ -14,7 +14,7 @@ class Parse: - def __init__(self, model: str = "th") -> None: + def __init__(self, model: Optional[str] = "th") -> None: if model is None: model = "th" self.nlp = esupar.load(model) diff --git a/pythainlp/parse/transformers_ud.py b/pythainlp/parse/transformers_ud.py index b3f432106..9d16780d8 100644 --- a/pythainlp/parse/transformers_ud.py +++ b/pythainlp/parse/transformers_ud.py @@ -12,12 +12,12 @@ from __future__ import annotations import os -from typing import Union +from typing import Optional, Union class Parse: def __init__( - self, model: str = "KoichiYasuoka/deberta-base-thai-ud-head" + self, model: Optional[str] = "KoichiYasuoka/deberta-base-thai-ud-head" ) -> None: from transformers import ( AutoConfig, diff --git a/pythainlp/parse/ud_goeswith.py b/pythainlp/parse/ud_goeswith.py index ae5c87395..4a4040936 100644 --- a/pythainlp/parse/ud_goeswith.py +++ b/pythainlp/parse/ud_goeswith.py @@ -11,7 +11,7 @@ from __future__ import annotations -from typing import List, Union +from typing import List, Optional, Union import numpy as np import torch @@ -21,7 +21,7 @@ class Parse: def __init__( - self, model: str = "KoichiYasuoka/deberta-base-thai-ud-goeswith" + self, model: Optional[str] = "KoichiYasuoka/deberta-base-thai-ud-goeswith" ) -> None: if model is None: model = "KoichiYasuoka/deberta-base-thai-ud-goeswith" diff --git a/pythainlp/translate/small100.py b/pythainlp/translate/small100.py index d7360b52a..d24b33ecd 100644 --- a/pythainlp/translate/small100.py +++ b/pythainlp/translate/small100.py @@ -68,7 +68,7 @@ def translate(self, text: str, tgt_lang: str = "en") -> str: self.translated = self.model.generate( **self.tokenizer(text, return_tensors="pt") ) - decoded_list = self.tokenizer.batch_decode( + decoded_list: list[str] = self.tokenizer.batch_decode( self.translated, skip_special_tokens=True ) return decoded_list[0] diff --git a/pythainlp/translate/th_fr.py b/pythainlp/translate/th_fr.py index 3e59c332e..40eb84297 100644 --- a/pythainlp/translate/th_fr.py +++ b/pythainlp/translate/th_fr.py @@ -63,7 +63,7 @@ def translate(self, text: str) -> str: self.translated = self.model_thzh.generate( **self.tokenizer_thzh(text, return_tensors="pt", padding=True) ) - decoded_list = [ + decoded_list: list[str] = [ self.tokenizer_thzh.decode(t, skip_special_tokens=True) for t in self.translated ] diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index 00cfdbaf6..6b29e37fe 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -218,10 +218,10 @@ def _convert_token_to_id(self, token: str) -> int: def _convert_id_to_token(self, index: int) -> str: """Converts an index (integer) in a token (str) using the decoder.""" if index in self.id_to_lang_token: - return self.id_to_lang_token[index] # type: ignore[no-any-return] + return self.id_to_lang_token[index] token = self.decoder.get(index, self.unk_token) if token is None: - return self.unk_token # type: ignore[no-any-return] + return self.unk_token return token # type: ignore[no-any-return] def convert_tokens_to_string(self, tokens: list[str]) -> str: @@ -257,7 +257,7 @@ def get_special_tokens_mask( token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True, - ) # type: ignore[no-any-return] + ) prefix_ones = [1] * len(self.prefix_tokens) if self.prefix_tokens else [] suffix_ones = [1] * len(self.suffix_tokens) diff --git a/pythainlp/translate/zh_th.py b/pythainlp/translate/zh_th.py index 5c5df3406..898be6c72 100644 --- a/pythainlp/translate/zh_th.py +++ b/pythainlp/translate/zh_th.py @@ -57,7 +57,7 @@ def translate(self, text: str) -> str: self.translated = self.model_thzh.generate( **self.tokenizer_thzh(text, return_tensors="pt", padding=True) ) - decoded_list = [ + decoded_list: list[str] = [ self.tokenizer_thzh.decode(t, skip_special_tokens=True) for t in self.translated ] @@ -109,7 +109,7 @@ def translate(self, text: str) -> str: self.translated = self.model_zhth.generate( **self.tokenizer_zhth(text, return_tensors="pt", padding=True) ) - decoded_list = [ + decoded_list: list[str] = [ self.tokenizer_zhth.decode(t, skip_special_tokens=True) for t in self.translated ] From 7ffb481a945d274813a03f1fd6e9886d1ae9e70a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Feb 2026 11:28:57 +0000 Subject: [PATCH 5/7] Fix more type hints - parse, chat, el, translate modules - Added Optional import to parse/esupar_engine.py - Fixed tokenization_small100.py type ignore comments - Added type ignore for external library calls in el/core.py and chat/core.py - Fixed parse/core.py type ignore comment error code - Reduced mypy errors from 46 to 40 Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/chat/core.py | 2 +- pythainlp/el/core.py | 2 +- pythainlp/parse/core.py | 2 +- pythainlp/parse/esupar_engine.py | 2 +- pythainlp/translate/tokenization_small100.py | 5 +++-- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pythainlp/chat/core.py b/pythainlp/chat/core.py index 18e2735a4..f77ded5c7 100644 --- a/pythainlp/chat/core.py +++ b/pythainlp/chat/core.py @@ -88,4 +88,4 @@ def chat(self, text: str) -> str: ) _bot = self.model.gen_instruct(_temp) self.history.append((text, _bot)) - return _bot + return _bot # type: ignore[no-any-return] diff --git a/pythainlp/el/core.py b/pythainlp/el/core.py index 679e3398a..6678b69b0 100644 --- a/pythainlp/el/core.py +++ b/pythainlp/el/core.py @@ -57,4 +57,4 @@ def get_el(self, list_text: Union[list[str], str]) -> Union[list[dict], str]: # 'md_scores': [0.30301809310913086, 0.6399497389793396], # 'el_scores': [0.7142490744590759, 0.8657019734382629]}] """ - return self.model.process_batch(list_text) + return self.model.process_batch(list_text) # type: ignore[no-any-return] diff --git a/pythainlp/parse/core.py b/pythainlp/parse/core.py index a024d6fa2..c105492a8 100644 --- a/pythainlp/parse/core.py +++ b/pythainlp/parse/core.py @@ -118,4 +118,4 @@ def dependency_parsing( _tagger_name = engine - return _tagger(text, tag=tag) # type: ignore[misc] + return _tagger(text, tag=tag) # type: ignore[no-any-return] diff --git a/pythainlp/parse/esupar_engine.py b/pythainlp/parse/esupar_engine.py index 33b59f14a..873c7a116 100644 --- a/pythainlp/parse/esupar_engine.py +++ b/pythainlp/parse/esupar_engine.py @@ -5,7 +5,7 @@ from __future__ import annotations -from typing import List, Union +from typing import List, Optional, Union try: import esupar diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index 6b29e37fe..9a4b0ca51 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -221,7 +221,7 @@ def _convert_id_to_token(self, index: int) -> str: return self.id_to_lang_token[index] token = self.decoder.get(index, self.unk_token) if token is None: - return self.unk_token + return self.unk_token # type: ignore[no-any-return] return token # type: ignore[no-any-return] def convert_tokens_to_string(self, tokens: list[str]) -> str: @@ -253,7 +253,8 @@ def get_special_tokens_mask( """ if already_has_special_tokens: - return super().get_special_tokens_mask( + # External library method + return super().get_special_tokens_mask( # type: ignore[no-any-return] token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True, From 808d6bdf9cea45d7ccf0386a65879921cfca141c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Feb 2026 11:39:46 +0000 Subject: [PATCH 6/7] Complete mypy type hints fixes and ruff formatting - Fixed all remaining 40 mypy errors (100% success - 0 errors!) - Added proper type annotations throughout the codebase - Fixed import issues in parse and tag modules - Updated pyproject.toml to ignore ufal module imports - Applied ruff formatting to all files - All type hints now properly verified Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pyproject.toml | 1 + pythainlp/__init__.py | 4 +- pythainlp/ancient/__init__.py | 3 +- pythainlp/augment/__init__.py | 3 +- pythainlp/augment/lm/__init__.py | 3 +- pythainlp/augment/lm/fasttext.py | 9 ++-- pythainlp/augment/lm/phayathaibert.py | 12 +++-- pythainlp/augment/word2vec/__init__.py | 3 +- pythainlp/augment/word2vec/bpemb_wv.py | 3 +- pythainlp/augment/word2vec/core.py | 5 +- pythainlp/augment/word2vec/ltw2v.py | 8 +-- pythainlp/augment/word2vec/thai2fit.py | 10 ++-- pythainlp/augment/wordnet.py | 19 +++---- pythainlp/benchmarks/__init__.py | 3 +- pythainlp/benchmarks/word_tokenization.py | 5 +- pythainlp/chat/__init__.py | 3 +- pythainlp/chat/core.py | 6 +-- pythainlp/classify/__init__.py | 3 +- pythainlp/cli/data.py | 3 +- pythainlp/cli/tag.py | 3 +- pythainlp/cli/tokenize.py | 4 +- pythainlp/coref/__init__.py | 3 +- pythainlp/coref/core.py | 4 +- pythainlp/corpus/common.py | 11 ++-- pythainlp/corpus/core.py | 43 +++++++++++----- pythainlp/corpus/icu.py | 3 +- pythainlp/corpus/oscar.py | 6 +-- pythainlp/corpus/th_en_translit.py | 10 ++-- pythainlp/corpus/tnc.py | 12 ++--- pythainlp/corpus/ttc.py | 3 +- pythainlp/corpus/util.py | 3 +- pythainlp/corpus/volubilis.py | 3 +- pythainlp/corpus/wikipedia.py | 3 +- pythainlp/corpus/wordnet.py | 20 ++++++-- pythainlp/el/__init__.py | 3 +- pythainlp/el/core.py | 4 +- pythainlp/generate/__init__.py | 3 +- pythainlp/generate/core.py | 4 +- pythainlp/morpheme/__init__.py | 3 +- pythainlp/parse/__init__.py | 3 +- pythainlp/parse/core.py | 14 +++-- pythainlp/parse/esupar_engine.py | 4 +- pythainlp/parse/spacy_thai_engine.py | 4 +- pythainlp/parse/transformers_ud.py | 6 ++- pythainlp/parse/ud_goeswith.py | 10 ++-- pythainlp/phayathaibert/__init__.py | 3 +- pythainlp/phayathaibert/core.py | 34 +++++++++---- pythainlp/soundex/complete_soundex.py | 8 ++- pythainlp/spell/__init__.py | 3 +- pythainlp/spell/core.py | 3 +- pythainlp/spell/pn.py | 13 +++-- pythainlp/spell/symspellpy.py | 4 +- pythainlp/spell/words_spelling_correction.py | 2 +- pythainlp/summarize/__init__.py | 3 +- pythainlp/summarize/core.py | 3 +- pythainlp/summarize/freq.py | 7 +-- pythainlp/summarize/keybert.py | 2 +- pythainlp/summarize/mt5.py | 3 +- pythainlp/tag/_tag_perceptron.py | 11 ++-- pythainlp/tag/crfchunk.py | 11 ++-- pythainlp/tag/locations.py | 3 +- pythainlp/tag/named_entity.py | 7 +-- pythainlp/tag/orchid.py | 3 +- pythainlp/tag/perceptron.py | 3 +- pythainlp/tag/thai_nner.py | 4 +- pythainlp/tag/thainer.py | 11 ++-- pythainlp/tag/unigram.py | 3 +- pythainlp/tag/wangchanberta_onnx.py | 14 +++-- pythainlp/tokenize/__init__.py | 4 +- pythainlp/tokenize/_utils.py | 9 ++-- pythainlp/tokenize/core.py | 25 ++++++--- pythainlp/tokenize/crfcut.py | 4 +- pythainlp/tokenize/deepcut.py | 4 +- pythainlp/tokenize/han_solo.py | 17 +++++-- pythainlp/tokenize/longest.py | 4 +- pythainlp/tokenize/multi_cut.py | 8 ++- pythainlp/tokenize/ssg.py | 3 +- pythainlp/tokenize/thaisumcut.py | 36 +++++++++---- pythainlp/tools/core.py | 3 +- pythainlp/tools/misspell.py | 17 +++++-- pythainlp/translate/__init__.py | 3 +- pythainlp/translate/core.py | 5 +- pythainlp/translate/en_th.py | 3 +- pythainlp/translate/tokenization_small100.py | 41 +++++++++++---- pythainlp/transliterate/__init__.py | 3 +- pythainlp/transliterate/lookup.py | 3 +- pythainlp/transliterate/thai2rom.py | 51 +++++++++++++------ pythainlp/transliterate/thai2rom_onnx.py | 6 +-- pythainlp/transliterate/thaig2p.py | 51 +++++++++++++------ pythainlp/transliterate/w2p.py | 4 +- pythainlp/ulmfit/core.py | 6 +-- pythainlp/ulmfit/preprocess.py | 3 +- pythainlp/ulmfit/tokenizer.py | 5 +- pythainlp/util/abbreviation.py | 3 +- pythainlp/util/date.py | 7 +-- pythainlp/util/digitconv.py | 3 +- pythainlp/util/emojiconv.py | 3 +- pythainlp/util/keyboard.py | 3 +- pythainlp/util/keywords.py | 4 +- pythainlp/util/normalize.py | 3 +- pythainlp/util/phoneme.py | 3 +- pythainlp/util/profanity.py | 1 + .../util/remove_trailing_repeat_consonants.py | 3 +- pythainlp/util/strftime.py | 4 +- pythainlp/util/syllable.py | 35 ++++++------- pythainlp/util/thai_lunar_date.py | 3 +- pythainlp/util/trie.py | 9 ++-- pythainlp/util/wordtonum.py | 4 +- pythainlp/wangchanberta/core.py | 26 +++++++--- pythainlp/wsd/__init__.py | 3 +- pythainlp/wsd/core.py | 6 +-- 111 files changed, 536 insertions(+), 358 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9f97d670e..d7d905cdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -433,6 +433,7 @@ module = [ "torch.*", "tqdm.*", "transformers.*", + "ufal.*", "ufal.chu_liu_edmonds.*", "word2word.*", "wtpsplit.*", diff --git a/pythainlp/__init__.py b/pythainlp/__init__.py index a41f1c374..36afd5c5f 100644 --- a/pythainlp/__init__.py +++ b/pythainlp/__init__.py @@ -14,14 +14,14 @@ thai_above_vowels = "\u0e31\u0e34\u0e35\u0e36\u0e37\u0e4d\u0e47" # 7 thai_below_vowels = "\u0e38\u0e39" # 2 -thai_tonemarks = "\u0e48\u0e49\u0e4a\u0e4b" # 4 +thai_tonemarks: str = "\u0e48\u0e49\u0e4a\u0e4b" # 4 # Paiyannoi, Maiyamok, Phinthu, Thanthakhat, Nikhahit, Yamakkan: # These signs can be part of a word thai_signs = "\u0e2f\u0e3a\u0e46\u0e4c\u0e4d\u0e4e" # 6 chars # Any Thai character that can be part of a word -thai_letters = "".join( +thai_letters: str = "".join( [thai_consonants, thai_vowels, thai_tonemarks, thai_signs] ) # 74 diff --git a/pythainlp/ancient/__init__.py b/pythainlp/ancient/__init__.py index 23a0f50ab..660baac1c 100644 --- a/pythainlp/ancient/__init__.py +++ b/pythainlp/ancient/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Ancient versions of the Thai language -""" +"""Ancient versions of the Thai language""" __all__ = ["aksonhan_to_current", "convert_currency"] diff --git a/pythainlp/augment/__init__.py b/pythainlp/augment/__init__.py index 79c7de2b3..5fb975bf4 100644 --- a/pythainlp/augment/__init__.py +++ b/pythainlp/augment/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Thai text augment -""" +"""Thai text augment""" __all__ = ["WordNetAug"] diff --git a/pythainlp/augment/lm/__init__.py b/pythainlp/augment/lm/__init__.py index 8c4e0cea2..40b3c32bc 100644 --- a/pythainlp/augment/lm/__init__.py +++ b/pythainlp/augment/lm/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Language Models -""" +"""Language Models""" __all__ = [ "FastTextAug", diff --git a/pythainlp/augment/lm/fasttext.py b/pythainlp/augment/lm/fasttext.py index 47dfea180..8ca099c01 100644 --- a/pythainlp/augment/lm/fasttext.py +++ b/pythainlp/augment/lm/fasttext.py @@ -15,8 +15,7 @@ class FastTextAug: """ def __init__(self, model_path: str): - """:param str model_path: path of model file - """ + """:param str model_path: path of model file""" from gensim.models.fasttext import FastText as FastText_gensim from gensim.models.keyedvectors import KeyedVectors @@ -38,8 +37,8 @@ def tokenize(self, text: str) -> list[str]: """ return word_tokenize(text, engine="icu") - def modify_sent(self, sent: str, p: float = 0.7) -> list[list[str]]: - """:param str sent: text of sentence + def modify_sent(self, sent: list[str], p: float = 0.7) -> list[list[str]]: + """:param list[str] sent: text of sentence :param float p: probability :rtype: List[List[str]] """ @@ -57,7 +56,7 @@ def modify_sent(self, sent: str, p: float = 0.7) -> list[list[str]]: def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> list[tuple[str]]: + ) -> list[tuple[str, ...]]: """Text Augment from fastText You may want to download the Thai model diff --git a/pythainlp/augment/lm/phayathaibert.py b/pythainlp/augment/lm/phayathaibert.py index c9dceb85e..cc0312312 100644 --- a/pythainlp/augment/lm/phayathaibert.py +++ b/pythainlp/augment/lm/phayathaibert.py @@ -20,7 +20,9 @@ def __init__(self) -> None: ) self.tokenizer = AutoTokenizer.from_pretrained(_MODEL_NAME) - self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained(_MODEL_NAME) + self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained( + _MODEL_NAME + ) self.model = pipeline( "fill-mask", tokenizer=self.tokenizer, @@ -53,7 +55,9 @@ def generate( return gen_txt - def augment(self, text: str, num_augs: int = 3, sample: bool = False) -> list[str]: + def augment( + self, text: str, num_augs: int = 3, sample: bool = False + ) -> list[str]: """Text augmentation from PhayaThaiBERT :param str text: Thai text @@ -87,7 +91,9 @@ def augment(self, text: str, num_augs: int = 3, sample: bool = False) -> list[st if num_augs <= MAX_NUM_AUGS: for rank in range(num_augs): gen_text = self.generate(text, rank, sample=sample) - processed_text = re.sub("<_>", " ", self.processor.preprocess(gen_text)) + processed_text = re.sub( + "<_>", " ", self.processor.preprocess(gen_text) + ) augment_list.append(processed_text) else: raise ValueError( diff --git a/pythainlp/augment/word2vec/__init__.py b/pythainlp/augment/word2vec/__init__.py index ff04e2cfc..1b27b47da 100644 --- a/pythainlp/augment/word2vec/__init__.py +++ b/pythainlp/augment/word2vec/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Word2Vec -""" +"""Word2Vec""" __all__ = ["Word2VecAug", "Thai2fitAug", "LTW2VAug"] diff --git a/pythainlp/augment/word2vec/bpemb_wv.py b/pythainlp/augment/word2vec/bpemb_wv.py index 8a9433f96..510551602 100644 --- a/pythainlp/augment/word2vec/bpemb_wv.py +++ b/pythainlp/augment/word2vec/bpemb_wv.py @@ -27,8 +27,7 @@ def tokenizer(self, text: str) -> list[str]: return self.bpemb_temp.encode(text) # type: ignore[no-any-return] def load_w2v(self): - """Load BPEmb model - """ + """Load BPEmb model""" self.aug = Word2VecAug( self.model, tokenize=self.tokenizer, type="model" ) diff --git a/pythainlp/augment/word2vec/core.py b/pythainlp/augment/word2vec/core.py index 224caf6ed..640703a56 100644 --- a/pythainlp/augment/word2vec/core.py +++ b/pythainlp/augment/word2vec/core.py @@ -9,7 +9,10 @@ class Word2VecAug: def __init__( - self, model: str, tokenize: Callable[[str], list[str]], type: str = "file" + self, + model: str, + tokenize: Callable[[str], list[str]], + type: str = "file", ) -> None: """:param str model: path of model :param Callable[[str], list[str]] tokenize: tokenize function diff --git a/pythainlp/augment/word2vec/ltw2v.py b/pythainlp/augment/word2vec/ltw2v.py index 679a44600..4be52f759 100644 --- a/pythainlp/augment/word2vec/ltw2v.py +++ b/pythainlp/augment/word2vec/ltw2v.py @@ -26,13 +26,13 @@ def tokenizer(self, text: str) -> list[str]: return word_tokenize(text, engine="newmm") def load_w2v(self): # insert substitute - """Load LTW2V's word2vec model - """ - self.aug = Word2VecAug(self.ltw2v_wv, self.tokenizer, type="binary") + """Load LTW2V's word2vec model""" + ltw2v_wv = self.ltw2v_wv or "" + self.aug = Word2VecAug(ltw2v_wv, self.tokenizer, type="binary") def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> list[tuple[str]]: + ) -> list[tuple[str, ...]]: """Text Augment using word2vec from Thai2Fit :param str sentence: Thai sentence diff --git a/pythainlp/augment/word2vec/thai2fit.py b/pythainlp/augment/word2vec/thai2fit.py index 6f0bd6af2..cb4a810d4 100644 --- a/pythainlp/augment/word2vec/thai2fit.py +++ b/pythainlp/augment/word2vec/thai2fit.py @@ -24,16 +24,16 @@ def tokenizer(self, text: str) -> list[str]: :rtype: List[str] """ tok = thai2fit_tokenizer() - return tok.word_tokenize(text) + return tok.word_tokenize(text) # type: ignore[no-any-return] def load_w2v(self): - """Load Thai2Fit's word2vec model - """ - self.aug = Word2VecAug(self.thai2fit_wv, self.tokenizer, type="binary") + """Load Thai2Fit's word2vec model""" + thai2fit_wv = self.thai2fit_wv or "" + self.aug = Word2VecAug(thai2fit_wv, self.tokenizer, type="binary") def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 - ) -> list[tuple[str]]: + ) -> list[tuple[str, ...]]: """Text Augment using word2vec from Thai2Fit :param str sentence: Thai sentence diff --git a/pythainlp/augment/wordnet.py b/pythainlp/augment/wordnet.py index b29efcc18..8ce64ed1d 100644 --- a/pythainlp/augment/wordnet.py +++ b/pythainlp/augment/wordnet.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Thank https://dev.to/ton_ami/text-data-augmentation-synonym-replacement-4h8l -""" +"""Thank https://dev.to/ton_ami/text-data-augmentation-synonym-replacement-4h8l""" from __future__ import annotations @@ -13,7 +12,7 @@ import itertools from collections import OrderedDict -from typing import Optional +from typing import Callable, Optional from nltk.corpus import wordnet as wn @@ -117,14 +116,16 @@ def postype2wordnet(pos: str, corpus: str): class WordNetAug: - """Text Augment using wordnet - """ + """Text Augment using wordnet""" def __init__(self): pass def find_synonyms( - self, word: str, pos: Optional[str] = None, postag_corpus: str = "orchid" + self, + word: str, + pos: Optional[str] = None, + postag_corpus: str = "orchid", ) -> list[str]: """Find synonyms using wordnet @@ -156,7 +157,7 @@ def find_synonyms( def augment( self, sentence: str, - tokenize: object = word_tokenize, + tokenize: Callable[[str], list[str]] = word_tokenize, max_syn_sent: int = 6, postag: bool = True, postag_corpus: str = "orchid", @@ -187,7 +188,7 @@ def augment( ('เรา', 'ชอบ', 'ไปยัง', 'รร.')] """ new_sentences = [] - self.list_words = tokenize(sentence) + self.list_words = word_tokenize(sentence) self.list_synonym = [] self.p_all = 1 if postag: @@ -210,5 +211,5 @@ def augment( if max_syn_sent > self.p_all: max_syn_sent = self.p_all for x in list(itertools.product(*self.list_synonym))[0:max_syn_sent]: - new_sentences.append(x) + new_sentences.append(list(x)) return new_sentences diff --git a/pythainlp/benchmarks/__init__.py b/pythainlp/benchmarks/__init__.py index c63536681..954076a96 100644 --- a/pythainlp/benchmarks/__init__.py +++ b/pythainlp/benchmarks/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Performance benchmarking. -""" +"""Performance benchmarking.""" __all__ = ["benchmark"] diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index 601f07090..647abca43 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -152,7 +152,10 @@ def compute_stats(ref_sample: str, raw_sample: str) -> dict: sample_arr = _binary_representation(raw_sample) # Compute character-level statistics - c_pos_pred, c_neg_pred = np.argwhere(sample_arr == 1), np.argwhere(sample_arr == 0) + c_pos_pred, c_neg_pred = ( + np.argwhere(sample_arr == 1), + np.argwhere(sample_arr == 0), + ) c_pos_pred = c_pos_pred[c_pos_pred < ref_sample_arr.shape[0]] c_neg_pred = c_neg_pred[c_neg_pred < ref_sample_arr.shape[0]] diff --git a/pythainlp/chat/__init__.py b/pythainlp/chat/__init__.py index 8becbc93d..10c25c49d 100644 --- a/pythainlp/chat/__init__.py +++ b/pythainlp/chat/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""pythainlp.chat -""" +"""pythainlp.chat""" __all__ = ["ChatBotModel"] diff --git a/pythainlp/chat/core.py b/pythainlp/chat/core.py index f77ded5c7..4795a43fd 100644 --- a/pythainlp/chat/core.py +++ b/pythainlp/chat/core.py @@ -8,13 +8,11 @@ class ChatBotModel: def __init__(self): - """Chat using AI generation - """ + """Chat using AI generation""" self.history = [] def reset_chat(self): - """Reset chat by cleaning history - """ + """Reset chat by cleaning history""" self.history = [] def load_model( diff --git a/pythainlp/classify/__init__.py b/pythainlp/classify/__init__.py index 16c3f3151..bdb012334 100644 --- a/pythainlp/classify/__init__.py +++ b/pythainlp/classify/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""pythainlp.classify -""" +"""pythainlp.classify""" __all__ = ["GzipModel"] diff --git a/pythainlp/cli/data.py b/pythainlp/cli/data.py index f20e4182e..7ff9fcef3 100644 --- a/pythainlp/cli/data.py +++ b/pythainlp/cli/data.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Command line for PyThaiNLP's dataset/corpus management. -""" +"""Command line for PyThaiNLP's dataset/corpus management.""" from __future__ import annotations diff --git a/pythainlp/cli/tag.py b/pythainlp/cli/tag.py index c10e1a74c..fe98406d9 100644 --- a/pythainlp/cli/tag.py +++ b/pythainlp/cli/tag.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Command line for PyThaiNLP's taggers. -""" +"""Command line for PyThaiNLP's taggers.""" from __future__ import annotations diff --git a/pythainlp/cli/tokenize.py b/pythainlp/cli/tokenize.py index 1f3b1acdd..653b77361 100644 --- a/pythainlp/cli/tokenize.py +++ b/pythainlp/cli/tokenize.py @@ -6,7 +6,7 @@ from __future__ import annotations import argparse -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from pythainlp import cli from pythainlp.tokenize import ( @@ -31,7 +31,7 @@ class SubAppBase: separator: str algorithm: str - run: Callable[..., list[str]] + run: Callable[..., Any] def __init__(self, name: str, argv: Sequence[str]) -> None: parser = argparse.ArgumentParser(**cli.make_usage("tokenize " + name)) # type: ignore[arg-type] diff --git a/pythainlp/coref/__init__.py b/pythainlp/coref/__init__.py index 9aa47e950..cc985376e 100644 --- a/pythainlp/coref/__init__.py +++ b/pythainlp/coref/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""PyThaiNLP Coreference Resolution -""" +"""PyThaiNLP Coreference Resolution""" __all__ = ["coreference_resolution"] diff --git a/pythainlp/coref/core.py b/pythainlp/coref/core.py index 77799f526..24da7554c 100644 --- a/pythainlp/coref/core.py +++ b/pythainlp/coref/core.py @@ -9,7 +9,9 @@ def coreference_resolution( - texts: Union[str, list[str]], model_name: str = "han-coref-v1.0", device: str = "cpu" + texts: Union[str, list[str]], + model_name: str = "han-coref-v1.0", + device: str = "cpu", ) -> list[dict]: """Coreference Resolution diff --git a/pythainlp/corpus/common.py b/pythainlp/corpus/common.py index e9bd468ee..e56a2b4b8 100644 --- a/pythainlp/corpus/common.py +++ b/pythainlp/corpus/common.py @@ -2,8 +2,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Common lists of words. -""" +"""Common lists of words.""" from __future__ import annotations @@ -86,7 +85,9 @@ def countries() -> frozenset[str]: return _THAI_COUNTRIES -def provinces(details: bool = False) -> Union[frozenset[str], list[dict[str, str]]]: +def provinces( + details: bool = False, +) -> Union[frozenset[str], list[dict[str, str]]]: """Return a frozenset of Thailand province names in Thai such as "กระบี่", "กรุงเทพมหานคร", "กาญจนบุรี", and "อุบลราชธานี". \n(See: `dev/pythainlp/corpus/thailand_provinces_th.txt\ @@ -222,7 +223,9 @@ def thai_profanity_words() -> frozenset[str]: """ global _THAI_PROFANITY_WORDS if not _THAI_PROFANITY_WORDS: - _THAI_PROFANITY_WORDS = get_corpus(_THAI_PROFANITY_WORDS_FILENAME, comments=False) + _THAI_PROFANITY_WORDS = get_corpus( + _THAI_PROFANITY_WORDS_FILENAME, comments=False + ) return _THAI_PROFANITY_WORDS diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index cbf8e4d34..8c952ac57 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -241,7 +241,9 @@ def get_corpus_default_db(name: str, version: str = "") -> Optional[str]: return None -def get_corpus_path(name: str, version: str = "", force: bool = False) -> Optional[str]: +def get_corpus_path( + name: str, version: str = "", force: bool = False +) -> Optional[str]: """Get corpus path. :param str name: corpus name @@ -401,9 +403,9 @@ def _is_within_directory(directory: str, target: str) -> bool: if not abs_directory.endswith(os.sep): abs_directory += os.sep - return abs_target.startswith(abs_directory) or abs_target == abs_directory.rstrip( - os.sep - ) + return abs_target.startswith( + abs_directory + ) or abs_target == abs_directory.rstrip(os.sep) def _safe_extract_tar(tar: tarfile.TarFile, path: str) -> None: @@ -437,7 +439,9 @@ def _safe_extract_tar(tar: tarfile.TarFile, path: str) -> None: # Check the member's target path member_path = os.path.join(path, member.name) if not _is_within_directory(path, member_path): - raise ValueError(f"Attempted path traversal in tar file: {member.name}") + raise ValueError( + f"Attempted path traversal in tar file: {member.name}" + ) # For symlinks, also validate the link target if member.issym() or member.islnk(): @@ -450,7 +454,9 @@ def _safe_extract_tar(tar: tarfile.TarFile, path: str) -> None: link_target = os.path.join(member_dir, link_target) else: # Absolute symlinks are dangerous - make them relative to extraction path - link_target = os.path.join(path, link_target.lstrip(os.sep)) + link_target = os.path.join( + path, link_target.lstrip(os.sep) + ) # Check if the resolved symlink target is within the directory if not _is_within_directory(path, link_target): @@ -496,7 +502,9 @@ def _safe_extract_zip(zip_file: zipfile.ZipFile, path: str) -> None: resolved_target = os.path.join(member_dir, link_target) else: # Absolute symlinks - make them relative to extraction path - resolved_target = os.path.join(path, link_target.lstrip(os.sep)) + resolved_target = os.path.join( + path, link_target.lstrip(os.sep) + ) # Check if the symlink target is within the directory if not _is_within_directory(path, resolved_target): @@ -568,7 +576,9 @@ def _check_version(cause: str) -> bool: return check -def download(name: str, force: bool = False, url: str = "", version: str = "") -> bool: +def download( + name: str, force: bool = False, url: str = "", version: str = "" +) -> bool: """Download corpus. The available corpus names can be seen in this file: @@ -626,7 +636,10 @@ def download(name: str, force: bool = False, url: str = "", version: str = "") - if version not in corpus["versions"]: print("Corpus not found.") return False - elif _check_version(corpus["versions"][version]["pythainlp_version"]) is False: + elif ( + _check_version(corpus["versions"][version]["pythainlp_version"]) + is False + ): print("Corpus version not supported.") return False corpus_versions = corpus["versions"][version] @@ -666,7 +679,9 @@ def download(name: str, force: bool = False, url: str = "", version: str = "") - foldername = name + "_" + str(version) if not os.path.exists(get_full_data_path(foldername)): os.mkdir(get_full_data_path(foldername)) - with zipfile.ZipFile(get_full_data_path(file_name), "r") as zip_file: + with zipfile.ZipFile( + get_full_data_path(file_name), "r" + ) as zip_file: _safe_extract_zip(zip_file, get_full_data_path(foldername)) if found: @@ -740,7 +755,9 @@ def remove(name: str) -> bool: return False with open(corpus_db_path(), encoding="utf-8-sig") as f: db = json.load(f) - data = [corpus for corpus in db["_default"].values() if corpus["name"] == name] + data = [ + corpus for corpus in db["_default"].values() if corpus["name"] == name + ] if data: path = get_corpus_path(name) @@ -841,5 +858,7 @@ def get_hf_hub(repo_id: str, filename: str = "") -> str: repo_id=repo_id, filename=filename, local_dir=root_project ) else: - output_path = snapshot_download(repo_id=repo_id, local_dir=root_project) + output_path = snapshot_download( + repo_id=repo_id, local_dir=root_project + ) return output_path # type: ignore[no-any-return] diff --git a/pythainlp/corpus/icu.py b/pythainlp/corpus/icu.py index 838ab2407..5a0d54c50 100644 --- a/pythainlp/corpus/icu.py +++ b/pythainlp/corpus/icu.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Provides an optional word list from International Components for Unicode (ICU) dictionary. -""" +"""Provides an optional word list from International Components for Unicode (ICU) dictionary.""" from __future__ import annotations diff --git a/pythainlp/corpus/oscar.py b/pythainlp/corpus/oscar.py index d9696e488..8475119ba 100644 --- a/pythainlp/corpus/oscar.py +++ b/pythainlp/corpus/oscar.py @@ -19,8 +19,7 @@ def word_freqs() -> list[tuple[str, int]]: - """Get word frequency from OSCAR Corpus (words tokenized using ICU) - """ + """Get word frequency from OSCAR Corpus (words tokenized using ICU)""" freqs: list[tuple[str, int]] = [] path = get_corpus_path(_OSCAR_FILENAME) if not path: @@ -41,8 +40,7 @@ def word_freqs() -> list[tuple[str, int]]: def unigram_word_freqs() -> dict[str, int]: - """Get unigram word frequency from OSCAR Corpus (words tokenized using ICU) - """ + """Get unigram word frequency from OSCAR Corpus (words tokenized using ICU)""" freqs: dict[str, int] = defaultdict(int) path = get_corpus_path(_OSCAR_FILENAME) if not path: diff --git a/pythainlp/corpus/th_en_translit.py b/pythainlp/corpus/th_en_translit.py index 7699c4510..2ac4b3a3d 100644 --- a/pythainlp/corpus/th_en_translit.py +++ b/pythainlp/corpus/th_en_translit.py @@ -25,7 +25,9 @@ TRANSLITERATE_FOLLOW_RTSG = "follow_rtsg" -def get_transliteration_dict() -> defaultdict[str, dict[str, list[Union[str, bool, None]]]]: +def get_transliteration_dict() -> defaultdict[ + str, dict[str, list[Union[str, bool, None]]] +]: """Get Thai to English transliteration dictionary. The returned dict is in dict[str, dict[List[str], List[Optional[bool]]]] format. @@ -40,8 +42,10 @@ def get_transliteration_dict() -> defaultdict[str, dict[str, list[Union[str, boo ) # use list, as one word can have multiple transliterations. - trans_dict: defaultdict[str, dict[str, list[Union[str, bool, None]]]] = defaultdict( - lambda: {TRANSLITERATE_EN: [], TRANSLITERATE_FOLLOW_RTSG: []} + trans_dict: defaultdict[str, dict[str, list[Union[str, bool, None]]]] = ( + defaultdict( + lambda: {TRANSLITERATE_EN: [], TRANSLITERATE_FOLLOW_RTSG: []} + ) ) try: text = corpus_file.read_text(encoding="utf-8") diff --git a/pythainlp/corpus/tnc.py b/pythainlp/corpus/tnc.py index 21e359b74..56c8836be 100644 --- a/pythainlp/corpus/tnc.py +++ b/pythainlp/corpus/tnc.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Thai National Corpus word frequency -""" +"""Thai National Corpus word frequency""" from __future__ import annotations @@ -39,8 +38,7 @@ def word_freqs() -> list[tuple[str, int]]: def unigram_word_freqs() -> dict[str, int]: - """Get unigram word frequency from Thai National Corpus (TNC) - """ + """Get unigram word frequency from Thai National Corpus (TNC)""" freqs: dict[str, int] = defaultdict(int) for line in get_corpus(_UNIGRAM_FILENAME): _temp = line.strip().split(" ") @@ -51,8 +49,7 @@ def unigram_word_freqs() -> dict[str, int]: def bigram_word_freqs() -> dict[tuple[str, str], int]: - """Get bigram word frequency from Thai National Corpus (TNC) - """ + """Get bigram word frequency from Thai National Corpus (TNC)""" freqs: dict[tuple[str, str], int] = defaultdict(int) path = get_corpus_path(_BIGRAM_CORPUS_NAME) if not path: @@ -72,8 +69,7 @@ def bigram_word_freqs() -> dict[tuple[str, str], int]: def trigram_word_freqs() -> dict[tuple[str, str, str], int]: - """Get trigram word frequency from Thai National Corpus (TNC) - """ + """Get trigram word frequency from Thai National Corpus (TNC)""" freqs: dict[tuple[str, str, str], int] = defaultdict(int) path = get_corpus_path(_TRIGRAM_CORPUS_NAME) if not path: diff --git a/pythainlp/corpus/ttc.py b/pythainlp/corpus/ttc.py index 908d9db74..81c3c244b 100644 --- a/pythainlp/corpus/ttc.py +++ b/pythainlp/corpus/ttc.py @@ -33,8 +33,7 @@ def word_freqs() -> list[tuple[str, int]]: def unigram_word_freqs() -> dict[str, int]: - """Get unigram word frequency from Thai Textbook Corpus (TTC) - """ + """Get unigram word frequency from Thai Textbook Corpus (TTC)""" freqs: dict[str, int] = defaultdict(int) for line in get_corpus(_UNIGRAM_FILENAME): diff --git a/pythainlp/corpus/util.py b/pythainlp/corpus/util.py index 61085fb05..d11e1f7dc 100644 --- a/pythainlp/corpus/util.py +++ b/pythainlp/corpus/util.py @@ -22,8 +22,7 @@ def index_pairs(words: list[str]) -> Iterator[tuple[int, int]]: - """Return beginning and ending indexes of word pairs - """ + """Return beginning and ending indexes of word pairs""" i = 0 for w in words: yield i, i + len(w) diff --git a/pythainlp/corpus/volubilis.py b/pythainlp/corpus/volubilis.py index 39f8c27ea..ed8993f3f 100644 --- a/pythainlp/corpus/volubilis.py +++ b/pythainlp/corpus/volubilis.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Provides an optional word list from the Volubilis dictionary. -""" +"""Provides an optional word list from the Volubilis dictionary.""" from __future__ import annotations diff --git a/pythainlp/corpus/wikipedia.py b/pythainlp/corpus/wikipedia.py index 2605286b2..e1d8e6f6c 100644 --- a/pythainlp/corpus/wikipedia.py +++ b/pythainlp/corpus/wikipedia.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Provides an optional word list from Thai Wikipedia titles. -""" +"""Provides an optional word list from Thai Wikipedia titles.""" from __future__ import annotations diff --git a/pythainlp/corpus/wordnet.py b/pythainlp/corpus/wordnet.py index a52828d3b..7f5eb6e78 100644 --- a/pythainlp/corpus/wordnet.py +++ b/pythainlp/corpus/wordnet.py @@ -30,7 +30,9 @@ from nltk.corpus import wordnet -def synsets(word: str, pos: Optional[str] = None, lang: str = "tha") -> list[wordnet.Synset]: +def synsets( + word: str, pos: Optional[str] = None, lang: str = "tha" +) -> list[wordnet.Synset]: """This function returns the synonym set for all lemmas of the given word with an optional argument to constrain the part of speech of the word. @@ -191,7 +193,9 @@ def langs() -> list[str]: return wordnet.langs() # type: ignore[no-any-return] -def lemmas(word: str, pos: Optional[str] = None, lang: str = "tha") -> list[wordnet.Lemma]: +def lemmas( + word: str, pos: Optional[str] = None, lang: str = "tha" +) -> list[wordnet.Lemma]: """This function returns all lemmas given the word with an optional argument to constrain the part of speech of the word. @@ -287,7 +291,9 @@ def lemma_from_key(key: str) -> wordnet.Lemma: return wordnet.lemma_from_key(key) -def path_similarity(synsets1: wordnet.Synset, synsets2: wordnet.Synset) -> float: +def path_similarity( + synsets1: wordnet.Synset, synsets2: wordnet.Synset +) -> float: """This function returns similarity between two synsets based on the shortest path distance calculated using the equation below. @@ -326,7 +332,9 @@ def path_similarity(synsets1: wordnet.Synset, synsets2: wordnet.Synset) -> float return wordnet.path_similarity(synsets1, synsets2) # type: ignore[no-any-return] -def lch_similarity(synsets1: wordnet.Synset, synsets2: wordnet.Synset) -> float: +def lch_similarity( + synsets1: wordnet.Synset, synsets2: wordnet.Synset +) -> float: """This function returns Leacock Chodorow similarity (LCH) between two synsets, based on the shortest path distance and the maximum depth of the taxonomy. The equation to @@ -363,7 +371,9 @@ def lch_similarity(synsets1: wordnet.Synset, synsets2: wordnet.Synset) -> float: return wordnet.lch_similarity(synsets1, synsets2) # type: ignore[no-any-return] -def wup_similarity(synsets1: wordnet.Synset, synsets2: wordnet.Synset) -> float: +def wup_similarity( + synsets1: wordnet.Synset, synsets2: wordnet.Synset +) -> float: """This function returns Wu-Palmer similarity (WUP) between two synsets, based on the depth of the two senses in the taxonomy and their Least Common Subsumer (most specific ancestor node). diff --git a/pythainlp/el/__init__.py b/pythainlp/el/__init__.py index 9638fa2fa..1a31a56bc 100644 --- a/pythainlp/el/__init__.py +++ b/pythainlp/el/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""pythainlp.el -""" +"""pythainlp.el""" __all__ = ["EntityLinker"] diff --git a/pythainlp/el/core.py b/pythainlp/el/core.py index 6678b69b0..9af8cdf7d 100644 --- a/pythainlp/el/core.py +++ b/pythainlp/el/core.py @@ -37,7 +37,9 @@ def __init__( self.model = MultiEL(model_name=self.model_name, device=self.device) - def get_el(self, list_text: Union[list[str], str]) -> Union[list[dict], str]: + def get_el( + self, list_text: Union[list[str], str] + ) -> Union[list[dict], str]: """Get Entity Linking from Thai Text :param str Union[List[str], str]: list of Thai text or text diff --git a/pythainlp/generate/__init__.py b/pythainlp/generate/__init__.py index 456d90467..4bd86f306 100644 --- a/pythainlp/generate/__init__.py +++ b/pythainlp/generate/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Thai Text Generation -""" +"""Thai Text Generation""" __all__ = ["Bigram", "Trigram", "Unigram"] diff --git a/pythainlp/generate/core.py b/pythainlp/generate/core.py index 0ce3f0633..34d9fa636 100644 --- a/pythainlp/generate/core.py +++ b/pythainlp/generate/core.py @@ -79,7 +79,9 @@ def gen_sentence( for i in self.word if self.counts[i] / self.n >= prob } - return self._next_word(rand_text, N, output_str, prob=prob, duplicate=duplicate) + return self._next_word( + rand_text, N, output_str, prob=prob, duplicate=duplicate + ) def _next_word( self, diff --git a/pythainlp/morpheme/__init__.py b/pythainlp/morpheme/__init__.py index b46042636..191985189 100644 --- a/pythainlp/morpheme/__init__.py +++ b/pythainlp/morpheme/__init__.py @@ -2,8 +2,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""PyThaiNLP morpheme -""" +"""PyThaiNLP morpheme""" __all__ = ["nighit", "is_native_thai"] from pythainlp.morpheme.thaiwordcheck import is_native_thai diff --git a/pythainlp/parse/__init__.py b/pythainlp/parse/__init__.py index 01fbdada9..4aca909a7 100644 --- a/pythainlp/parse/__init__.py +++ b/pythainlp/parse/__init__.py @@ -2,8 +2,7 @@ # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""PyThaiNLP Parse -""" +"""PyThaiNLP Parse""" __all__ = ["dependency_parsing"] diff --git a/pythainlp/parse/core.py b/pythainlp/parse/core.py index c105492a8..b02e0273c 100644 --- a/pythainlp/parse/core.py +++ b/pythainlp/parse/core.py @@ -104,7 +104,11 @@ def dependency_parsing( elif engine == "transformers_ud": from pythainlp.parse.transformers_ud import Parse # type: ignore[assignment] # noqa: I001 - _tagger = Parse(model=model if model else "KoichiYasuoka/deberta-base-thai-ud-head") + _tagger = Parse( + model=model + if model + else "KoichiYasuoka/deberta-base-thai-ud-head" + ) elif engine == "spacy_thai": from pythainlp.parse.spacy_thai_engine import Parse # type: ignore[assignment] # noqa: I001 @@ -112,10 +116,14 @@ def dependency_parsing( elif engine == "ud_goeswith": from pythainlp.parse.ud_goeswith import Parse # type: ignore[assignment] # noqa: I001 - _tagger = Parse(model=model if model else "KoichiYasuoka/deberta-base-thai-ud-goeswith") + _tagger = Parse( + model=model + if model + else "KoichiYasuoka/deberta-base-thai-ud-goeswith" + ) else: raise NotImplementedError("The engine doesn't support.") _tagger_name = engine - return _tagger(text, tag=tag) # type: ignore[no-any-return] + return _tagger(text, tag=tag) # type: ignore[misc,no-any-return] diff --git a/pythainlp/parse/esupar_engine.py b/pythainlp/parse/esupar_engine.py index 873c7a116..9dd875f03 100644 --- a/pythainlp/parse/esupar_engine.py +++ b/pythainlp/parse/esupar_engine.py @@ -19,7 +19,9 @@ def __init__(self, model: Optional[str] = "th") -> None: model = "th" self.nlp = esupar.load(model) - def __call__(self, text: str, tag: str = "str") -> Union[List[List[str]], str]: + def __call__( + self, text: str, tag: str = "str" + ) -> Union[List[List[str]], str]: _data = str(self.nlp(text)) if tag == "list": _temp = _data.splitlines() diff --git a/pythainlp/parse/spacy_thai_engine.py b/pythainlp/parse/spacy_thai_engine.py index 9402b197a..cffae5cc5 100644 --- a/pythainlp/parse/spacy_thai_engine.py +++ b/pythainlp/parse/spacy_thai_engine.py @@ -15,7 +15,9 @@ class Parse: def __init__(self, model: str = "th") -> None: self.nlp = spacy_thai.load() - def __call__(self, text: str, tag: str = "str") -> Union[List[List[str]], str]: + def __call__( + self, text: str, tag: str = "str" + ) -> Union[List[List[str]], str]: doc = self.nlp(text) _text = [] if tag == "list": diff --git a/pythainlp/parse/transformers_ud.py b/pythainlp/parse/transformers_ud.py index 9d16780d8..076de9e85 100644 --- a/pythainlp/parse/transformers_ud.py +++ b/pythainlp/parse/transformers_ud.py @@ -54,7 +54,9 @@ def __init__( model=t, tokenizer=self.tokenizer ) - def __call__(self, text: str, tag: str = "str") -> Union[list[list[str]], str]: + def __call__( + self, text: str, tag: str = "str" + ) -> Union[list[list[str]], str]: import numpy import torch import ufal.chu_liu_edmonds @@ -102,7 +104,7 @@ def __call__(self, text: str, tag: str = "str") -> Union[list[list[str]], str]: h = ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0] if [0 for i in h if i == 0] != [0]: i = ([p for s, e, p in w] + ["root"]).index("root") - j = i + 1 if i < n else numpy.nanargmax(m[:, 0]) + j = i + 1 if i < n else int(numpy.nanargmax(m[:, 0])) m[0:j, 0] = m[j + 1 :, 0] = numpy.nan h = ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0] u = "" diff --git a/pythainlp/parse/ud_goeswith.py b/pythainlp/parse/ud_goeswith.py index 4a4040936..5ca1c2d81 100644 --- a/pythainlp/parse/ud_goeswith.py +++ b/pythainlp/parse/ud_goeswith.py @@ -15,20 +15,24 @@ import numpy as np import torch -import ufal.chu_liu_edmonds from transformers import AutoModelForTokenClassification, AutoTokenizer class Parse: def __init__( - self, model: Optional[str] = "KoichiYasuoka/deberta-base-thai-ud-goeswith" + self, + model: Optional[str] = "KoichiYasuoka/deberta-base-thai-ud-goeswith", ) -> None: if model is None: model = "KoichiYasuoka/deberta-base-thai-ud-goeswith" self.tokenizer = AutoTokenizer.from_pretrained(model) self.model = AutoModelForTokenClassification.from_pretrained(model) - def __call__(self, text: str, tag: str = "str") -> Union[List[List[str]], str]: + def __call__( + self, text: str, tag: str = "str" + ) -> Union[List[List[str]], str]: + import ufal.chu_liu_edmonds + w = self.tokenizer(text, return_offsets_mapping=True) v = w["input_ids"] x = [ diff --git a/pythainlp/phayathaibert/__init__.py b/pythainlp/phayathaibert/__init__.py index c2a2e24b2..657730ab2 100644 --- a/pythainlp/phayathaibert/__init__.py +++ b/pythainlp/phayathaibert/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""PhayaThaiBERT -""" +"""PhayaThaiBERT""" __all__ = [ "NamedEntityTagger", diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index 81f059c23..176ba833c 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -61,13 +61,25 @@ def rm_brackets(self, text: str) -> str: new_line = re.sub(r"\{[^a-zA-Z0-9ก-๙]+\}", "", new_line) new_line = re.sub(r"\[[^a-zA-Z0-9ก-๙]+\]", "", new_line) # artifiacts after ( - new_line = re.sub(r"(?<=\()[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line) - new_line = re.sub(r"(?<=\{)[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line) - new_line = re.sub(r"(?<=\[)[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line) + new_line = re.sub( + r"(?<=\()[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line + ) + new_line = re.sub( + r"(?<=\{)[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line + ) + new_line = re.sub( + r"(?<=\[)[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line + ) # artifacts before ) - new_line = re.sub(r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\))", "", new_line) - new_line = re.sub(r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\})", "", new_line) - new_line = re.sub(r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\])", "", new_line) + new_line = re.sub( + r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\))", "", new_line + ) + new_line = re.sub( + r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\})", "", new_line + ) + new_line = re.sub( + r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\])", "", new_line + ) return new_line def replace_newlines(self, text: str) -> str: @@ -195,7 +207,9 @@ def __init__(self) -> None: ) self.tokenizer = AutoTokenizer.from_pretrained(_model_name) - self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained(_model_name) + self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained( + _model_name + ) self.model = pipeline( "fill-mask", tokenizer=self.tokenizer, @@ -268,7 +282,9 @@ def augment( rank, sample=sample, ) - processed_text = re.sub("<_>", " ", self.processor.preprocess(gen_text)) + processed_text = re.sub( + "<_>", " ", self.processor.preprocess(gen_text) + ) augment_list.append(processed_text) else: raise ValueError( @@ -422,4 +438,4 @@ def segment(sentence: str) -> list[str]: if not sentence or not isinstance(sentence, str): return [] - return _tokenizer.tokenize(sentence) + return _tokenizer.tokenize(sentence) # type: ignore[no-any-return] diff --git a/pythainlp/soundex/complete_soundex.py b/pythainlp/soundex/complete_soundex.py index d03684f33..3dc63e7e2 100644 --- a/pythainlp/soundex/complete_soundex.py +++ b/pythainlp/soundex/complete_soundex.py @@ -28,6 +28,7 @@ complete_soundex("ปุญญา") # 'ปป4G0น-ยย1B0--*' complete_soundex("สวรรค์") # 'ซศ1A-0-วว1Aน0-' """ + from __future__ import annotations import re @@ -682,11 +683,14 @@ def complete_soundex_similarity(code1: str, code2: str) -> float: :Example: :: - from pythainlp.soundex import complete_soundex, complete_soundex_similarity + from pythainlp.soundex import ( + complete_soundex, + complete_soundex_similarity, + ) # Encode two words code1 = complete_soundex("ข้มขืน") # Bitter/Forced (with tone) - code2 = complete_soundex("ขมขืน") # Bitter (no tone) + code2 = complete_soundex("ขมขืน") # Bitter (no tone) # Calculate similarity similarity = complete_soundex_similarity(code1, code2) diff --git a/pythainlp/spell/__init__.py b/pythainlp/spell/__init__.py index 8850ceb62..36ab4b132 100644 --- a/pythainlp/spell/__init__.py +++ b/pythainlp/spell/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Spell checking and correction. -""" +"""Spell checking and correction.""" __all__ = [ "DEFAULT_SPELL_CHECKER", diff --git a/pythainlp/spell/core.py b/pythainlp/spell/core.py index c4bbeaec5..d32c784fa 100644 --- a/pythainlp/spell/core.py +++ b/pythainlp/spell/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Spell checking functions -""" +"""Spell checking functions""" from __future__ import annotations diff --git a/pythainlp/spell/pn.py b/pythainlp/spell/pn.py index 5c5f28f2b..f869fd6bb 100644 --- a/pythainlp/spell/pn.py +++ b/pythainlp/spell/pn.py @@ -54,8 +54,7 @@ def _keep( def _edits1(word: str) -> set[str]: - """Returns a set of words with an edit distance of 1 from the input word - """ + """Returns a set of words with an edit distance of 1 from the input word""" splits = [(word[:i], word[i:]) for i in range(len(word) + 1)] deletes = [L + R[1:] for L, R in splits if R] transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R) > 1] @@ -72,20 +71,20 @@ def _edits1(word: str) -> set[str]: def _edits2(word: str) -> set[str]: - """Returns a set of words with an edit distance of 2 from the input word - """ + """Returns a set of words with an edit distance of 2 from the input word""" return set(e2 for e1 in _edits1(word) for e2 in _edits1(e1)) def _convert_custom_dict( - custom_dict: Union[dict[str, int], Iterable[str], Iterable[tuple[str, int]]], + custom_dict: Union[ + dict[str, int], Iterable[str], Iterable[tuple[str, int]] + ], min_freq: int, min_len: int, max_len: int, dict_filter: Optional[Callable[[str], bool]], ) -> list[tuple[str, int]]: - """Converts a custom dictionary to a list of (str, int) tuples - """ + """Converts a custom dictionary to a list of (str, int) tuples""" if isinstance(custom_dict, dict): custom_dict = list(custom_dict.items()) diff --git a/pythainlp/spell/symspellpy.py b/pythainlp/spell/symspellpy.py index 57d02d8ce..7064fbf60 100644 --- a/pythainlp/spell/symspellpy.py +++ b/pythainlp/spell/symspellpy.py @@ -29,7 +29,9 @@ _BIGRAM_CORPUS_NAME = "tnc_bigram_word_freqs" _sym_spell = None -_unigram_file_ctx = None # File context manager kept alive for program lifetime +_unigram_file_ctx = ( + None # File context manager kept alive for program lifetime +) _load_lock = threading.Lock() # Thread safety for lazy loading diff --git a/pythainlp/spell/words_spelling_correction.py b/pythainlp/spell/words_spelling_correction.py index fde50168b..82f43e9e4 100644 --- a/pythainlp/spell/words_spelling_correction.py +++ b/pythainlp/spell/words_spelling_correction.py @@ -272,4 +272,4 @@ def get_words_spell_suggestion( global _WSC if _WSC is None: _WSC = Words_Spelling_Correction() - return _WSC.get_word_suggestion(list_words) + return _WSC.get_word_suggestion(list_words) # type: ignore[no-any-return] diff --git a/pythainlp/summarize/__init__.py b/pythainlp/summarize/__init__.py index 6294498aa..6265d58ed 100644 --- a/pythainlp/summarize/__init__.py +++ b/pythainlp/summarize/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Text summarization -""" +"""Text summarization""" __all__ = [ "extract_keywords", diff --git a/pythainlp/summarize/core.py b/pythainlp/summarize/core.py index f58204d53..f0c106ff2 100644 --- a/pythainlp/summarize/core.py +++ b/pythainlp/summarize/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Text summarization and keyword extraction -""" +"""Text summarization and keyword extraction""" from __future__ import annotations diff --git a/pythainlp/summarize/freq.py b/pythainlp/summarize/freq.py index 615c9982d..5b9485047 100644 --- a/pythainlp/summarize/freq.py +++ b/pythainlp/summarize/freq.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Summarization by frequency of words -""" +"""Summarization by frequency of words""" from __future__ import annotations @@ -51,7 +50,9 @@ def summarize( self, text: str, n: int, tokenizer: str = "newmm" ) -> list[str]: # sent_tokenize with str input returns list[str] - sents = cast(list[str], sent_tokenize(text, engine="whitespace+newline")) + sents = cast( + list[str], sent_tokenize(text, engine="whitespace+newline") + ) word_tokenized_sents = [ word_tokenize(sent, engine=tokenizer) for sent in sents ] diff --git a/pythainlp/summarize/keybert.py b/pythainlp/summarize/keybert.py index cec7f6fc1..ff7773d2e 100644 --- a/pythainlp/summarize/keybert.py +++ b/pythainlp/summarize/keybert.py @@ -222,7 +222,7 @@ def l2_norm(v: np.ndarray) -> np.ndarray: ) return result # type: ignore[no-any-return] - def cosine_sim(a: np.ndarray, b: np.ndarray) -> np.ndarray: # type: ignore[type-arg] + def cosine_sim(a: np.ndarray, b: np.ndarray) -> np.ndarray: return (np.matmul(a, b.T).T).sum(axis=1) # type: ignore[no-any-return] doc_vector = l2_norm(doc_vector) diff --git a/pythainlp/summarize/mt5.py b/pythainlp/summarize/mt5.py index 9cf21487b..f88597c5e 100644 --- a/pythainlp/summarize/mt5.py +++ b/pythainlp/summarize/mt5.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Summarization by mT5 model -""" +"""Summarization by mT5 model""" from __future__ import annotations diff --git a/pythainlp/tag/_tag_perceptron.py b/pythainlp/tag/_tag_perceptron.py index 2eef5a73d..cbdc3faef 100644 --- a/pythainlp/tag/_tag_perceptron.py +++ b/pythainlp/tag/_tag_perceptron.py @@ -61,7 +61,9 @@ def predict(self, features: dict[str, float]) -> str: # Do a secondary alphabetic sort, for stability return max(self.classes, key=lambda label: (scores[label], label)) - def update(self, truth: str, guess: str, features: dict[str, float]) -> None: + def update( + self, truth: str, guess: str, features: dict[str, float] + ) -> None: """Update the feature weights.""" def upd_feat(c: str, f: str, w: float, v: float) -> None: @@ -118,8 +120,7 @@ class PerceptronTagger: AP_MODEL_LOC = "" def __init__(self, path: str = "") -> None: - """:param str path: model path - """ + """:param str path: model path""" self.model = AveragedPerceptron() self.tagdict: dict[str, str] = {} self.classes: set[str] = set() @@ -264,7 +265,9 @@ def _make_tagdict( self, sentences: Iterable[Iterable[tuple[str, str]]] ) -> None: """Make a tag dictionary for single-tag words.""" - counts: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int)) + counts: dict[str, dict[str, int]] = defaultdict( + lambda: defaultdict(int) + ) for sentence in sentences: for word, tag in sentence: counts[word][tag] += 1 diff --git a/pythainlp/tag/crfchunk.py b/pythainlp/tag/crfchunk.py index ef760fbeb..0748593f7 100644 --- a/pythainlp/tag/crfchunk.py +++ b/pythainlp/tag/crfchunk.py @@ -4,6 +4,7 @@ from __future__ import annotations import types +from contextlib import AbstractContextManager from importlib.resources import as_file, files from typing import Any, Optional @@ -71,6 +72,8 @@ class CRFchunk: garbage collected, though this is not guaranteed. """ + _model_file_ctx: Optional[AbstractContextManager[Any]] + def __init__(self, corpus: str = "orchidpp"): self.corpus = corpus self._model_file_ctx = None @@ -81,13 +84,13 @@ def load_model(self, corpus: str) -> None: if corpus == "orchidpp": corpus_files = files("pythainlp.corpus") model_file = corpus_files.joinpath("crfchunk_orchidpp.model") - self._model_file_ctx = as_file(model_file) # type: ignore[assignment] - model_path = self._model_file_ctx.__enter__() # type: ignore[attr-defined] + self._model_file_ctx = as_file(model_file) + model_path = self._model_file_ctx.__enter__() self.tagger.open(str(model_path)) def parse(self, token_pos: list[tuple[str, str]]) -> list[str]: self.xseq = extract_features(token_pos) - return self.tagger.tag(self.xseq) + return self.tagger.tag(self.xseq) # type: ignore[no-any-return] def __enter__(self) -> CRFchunk: """Context manager entry.""" @@ -97,7 +100,7 @@ def __exit__( self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], - exc_tb: Optional[types.TracebackType] + exc_tb: Optional[types.TracebackType], ) -> None: """Context manager exit - clean up resources.""" if self._model_file_ctx is not None: diff --git a/pythainlp/tag/locations.py b/pythainlp/tag/locations.py index 9b2143c0a..1a4802423 100644 --- a/pythainlp/tag/locations.py +++ b/pythainlp/tag/locations.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Recognizes locations in text -""" +"""Recognizes locations in text""" from __future__ import annotations diff --git a/pythainlp/tag/named_entity.py b/pythainlp/tag/named_entity.py index e1d6775bb..d9c0c3470 100644 --- a/pythainlp/tag/named_entity.py +++ b/pythainlp/tag/named_entity.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Named-entity recognizer -""" +"""Named-entity recognizer""" from __future__ import annotations @@ -49,7 +48,9 @@ def load_engine(self, engine: str, corpus: str) -> None: model="pythainlp/thainer-corpus-v2-base-model" ) elif engine == "wangchanberta": - from pythainlp.wangchanberta import ThaiNameTagger as WangchanbertaThaiNameTagger # noqa: I001,E501 + from pythainlp.wangchanberta import ( + ThaiNameTagger as WangchanbertaThaiNameTagger, + ) # noqa: I001,E501 self.engine = WangchanbertaThaiNameTagger(dataset_name=corpus) elif corpus == "thainer-v2": diff --git a/pythainlp/tag/orchid.py b/pythainlp/tag/orchid.py index f9a905a57..3344e051a 100644 --- a/pythainlp/tag/orchid.py +++ b/pythainlp/tag/orchid.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Data preprocessing for ORCHID corpus -""" +"""Data preprocessing for ORCHID corpus""" from __future__ import annotations diff --git a/pythainlp/tag/perceptron.py b/pythainlp/tag/perceptron.py index 4c7a3b004..fa2cfe436 100644 --- a/pythainlp/tag/perceptron.py +++ b/pythainlp/tag/perceptron.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Perceptron part-of-speech tagger -""" +"""Perceptron part-of-speech tagger""" from __future__ import annotations diff --git a/pythainlp/tag/thai_nner.py b/pythainlp/tag/thai_nner.py index 09432e7f1..98fc4ea8f 100644 --- a/pythainlp/tag/thai_nner.py +++ b/pythainlp/tag/thai_nner.py @@ -12,5 +12,5 @@ class Thai_NNER: def __init__(self, path_model=get_corpus_path("thai_nner", "1.0")) -> None: self.model = NNER(path_model=path_model) - def tag(self, text) -> tuple[list[str], list[dict]]: - return self.model.get_tag(text) + def tag(self, text) -> tuple[list[str], list[dict[str, str]]]: + return self.model.get_tag(text) # type: ignore[no-any-return] diff --git a/pythainlp/tag/thainer.py b/pythainlp/tag/thainer.py index f943e3d52..081de29cf 100644 --- a/pythainlp/tag/thainer.py +++ b/pythainlp/tag/thainer.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Named-entity recognizer -""" +"""Named-entity recognizer""" from __future__ import annotations @@ -12,7 +11,7 @@ from typing import Union from pythainlp.corpus import get_corpus_path, thai_stopwords -from pythainlp.tag import pos_tag +from pythainlp.tag.pos_tag import pos_tag from pythainlp.tokenize import word_tokenize from pythainlp.util import isthai @@ -124,7 +123,7 @@ def __init__(self, version: str = "1.4") -> None: def get_ner( self, text: str, pos: bool = True, tag: bool = False - ) -> Union[list[tuple[str, str]], list[tuple[str, str, str]]]: + ) -> Union[list[tuple[str, str]], list[tuple[str, str, str]], str]: """This function tags named-entities in text in IOB format. :param str text: text in Thai to be tagged @@ -214,5 +213,7 @@ def get_ner( return sent_ner @staticmethod - def __extract_features(doc: list[str]) -> list[dict[str, Union[str, bool]]]: + def __extract_features( + doc: list[tuple[str, str]], + ) -> list[dict[str, Union[str, bool]]]: return [_doc2features(doc, i) for i in range(len(doc))] diff --git a/pythainlp/tag/unigram.py b/pythainlp/tag/unigram.py index 13fab6120..628bf7682 100644 --- a/pythainlp/tag/unigram.py +++ b/pythainlp/tag/unigram.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Unigram Part-Of-Speech tagger -""" +"""Unigram Part-Of-Speech tagger""" from __future__ import annotations diff --git a/pythainlp/tag/wangchanberta_onnx.py b/pythainlp/tag/wangchanberta_onnx.py index 3d3e8f98b..bf2568698 100644 --- a/pythainlp/tag/wangchanberta_onnx.py +++ b/pythainlp/tag/wangchanberta_onnx.py @@ -69,9 +69,11 @@ def postprocess(self, logits_data: np.ndarray) -> np.ndarray: maxes = np.max(logits_t, axis=-1, keepdims=True) shifted_exp = np.exp(logits_t - maxes) scores = shifted_exp / shifted_exp.sum(axis=-1, keepdims=True) - return scores + return scores # type: ignore[no-any-return] - def clean_output(self, list_text: list[tuple[str, str]]) -> list[tuple[str, str]]: + def clean_output( + self, list_text: list[tuple[str, str]] + ) -> list[tuple[str, str]]: return list_text def totag(self, post: np.ndarray, sent: str) -> list[tuple[str, str]]: @@ -88,10 +90,14 @@ def totag(self, post: np.ndarray, sent: str) -> list[tuple[str, str]]: ) return tag - def _config(self, list_ner: list[tuple[str, str]]) -> list[tuple[str, str]]: + def _config( + self, list_ner: list[tuple[str, str]] + ) -> list[tuple[str, str]]: return list_ner - def get_ner(self, text: str, tag: bool = False) -> Union[str, list[tuple[str, str]]]: + def get_ner( + self, text: str, tag: bool = False + ) -> Union[str, list[tuple[str, str]]]: self._s = self.build_tokenizer(text) logits = self.session.run( output_names=[self.outputs_name], input_feed=self._s diff --git a/pythainlp/tokenize/__init__.py b/pythainlp/tokenize/__init__.py index 9799dd213..08c24fe29 100644 --- a/pythainlp/tokenize/__init__.py +++ b/pythainlp/tokenize/__init__.py @@ -1,8 +1,8 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Tokenizers at different levels of linguistic analysis. -""" +"""Tokenizers at different levels of linguistic analysis.""" + from __future__ import annotations __all__ = [ diff --git a/pythainlp/tokenize/_utils.py b/pythainlp/tokenize/_utils.py index c9aa0f523..6fa67de17 100644 --- a/pythainlp/tokenize/_utils.py +++ b/pythainlp/tokenize/_utils.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Utility functions for tokenize module. -""" +"""Utility functions for tokenize module.""" from __future__ import annotations @@ -13,10 +12,10 @@ def apply_postprocessors( - segments: list[str], postprocessors: Sequence[Callable[[list[str]], list[str]]] + segments: list[str], + postprocessors: Sequence[Callable[[list[str]], list[str]]], ) -> list[str]: - """A list of callables to apply to a raw segmentation result. - """ + """A list of callables to apply to a raw segmentation result.""" for func in postprocessors: segments = func(segments) diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index 5a7aa1cb9..54ccb65a6 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Generic functions of tokenizers -""" +"""Generic functions of tokenizers""" from __future__ import annotations @@ -352,10 +351,10 @@ def indices_words(words: list[str]) -> list[tuple[int, int]]: from pythainlp.tokenize import indices_words - indices_words(['สวัสดี', 'ครับ']) + indices_words(["สวัสดี", "ครับ"]) # output: [(0, 5), (6, 9)] - indices_words(['hello', 'world']) + indices_words(["hello", "world"]) # output: [(0, 4), (5, 9)] """ indices = [] @@ -368,7 +367,9 @@ def indices_words(words: list[str]) -> list[tuple[int, int]]: return indices -def map_indices_to_words(index_list: list[tuple[int, int]], sentences: list[str]) -> list[list[str]]: +def map_indices_to_words( + index_list: list[tuple[int, int]], sentences: list[str] +) -> list[list[str]]: """Map character index pairs to actual words from sentences. This function takes a list of character index pairs and a list of @@ -385,7 +386,7 @@ def map_indices_to_words(index_list: list[tuple[int, int]], sentences: list[str] from pythainlp.tokenize import map_indices_to_words indices = [(0, 5), (6, 9)] - sentences = ['สวัสดีครับ'] + sentences = ["สวัสดีครับ"] map_indices_to_words(indices, sentences) # output: [['สวัสดี', 'ครับ']] """ @@ -549,7 +550,9 @@ def sent_tokenize( _size = engine.split("-")[-1] from pythainlp.tokenize.wtsplit import tokenize - segments = tokenize(text=original_text, size=_size, tokenize="sentence") + segments = tokenize( + text=original_text, size=_size, tokenize="sentence" + ) else: raise ValueError( f"""Tokenizer \"{engine}\" not found. @@ -720,15 +723,19 @@ def subword_tokenize( if engine == "tcc": from pythainlp.tokenize.tcc import segment as tcc_segment + segments = tcc_segment(text) elif engine == "tcc_p": from pythainlp.tokenize.tcc_p import segment as tcc_p_segment + segments = tcc_p_segment(text) elif engine == "etcc": from pythainlp.tokenize.etcc import segment as etcc_segment + segments = etcc_segment(text) elif engine == "wangchanberta": from pythainlp.wangchanberta import segment as wangchanberta_segment + segments = wangchanberta_segment(text) elif engine == "dict": # use syllable dictionary words = word_tokenize(text) @@ -738,15 +745,19 @@ def subword_tokenize( ) elif engine == "ssg": from pythainlp.tokenize.ssg import segment as ssg_segment + segments = ssg_segment(text) elif engine == "tltk": from pythainlp.tokenize.tltk import syllable_tokenize as tltk_segment + segments = tltk_segment(text) elif engine == "han_solo": from pythainlp.tokenize.han_solo import segment as han_solo_segment + segments = han_solo_segment(text) elif engine == "phayathai": from pythainlp.phayathaibert import segment as phayathai_segment + segments = phayathai_segment(text) else: raise ValueError( diff --git a/pythainlp/tokenize/crfcut.py b/pythainlp/tokenize/crfcut.py index 1150bb0a9..522fdd313 100644 --- a/pythainlp/tokenize/crfcut.py +++ b/pythainlp/tokenize/crfcut.py @@ -150,7 +150,9 @@ def extract_features( # add enders and starters doc_ender = ["ender" if token in _ENDERS else "normal" for token in doc] - doc_starter = ["starter" if token in _STARTERS else "normal" for token in doc] + doc_starter = [ + "starter" if token in _STARTERS else "normal" for token in doc + ] # for each word for i in range(window, len(doc) - window): diff --git a/pythainlp/tokenize/deepcut.py b/pythainlp/tokenize/deepcut.py index 30a491670..8fda7477b 100644 --- a/pythainlp/tokenize/deepcut.py +++ b/pythainlp/tokenize/deepcut.py @@ -21,7 +21,9 @@ from pythainlp.util import Trie -def segment(text: str, custom_dict: Union[Trie, list[str], str] = []) -> list[str]: +def segment( + text: str, custom_dict: Union[Trie, list[str], str] = [] +) -> list[str]: if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/han_solo.py b/pythainlp/tokenize/han_solo.py index 686c81ee2..96a25eec7 100644 --- a/pythainlp/tokenize/han_solo.py +++ b/pythainlp/tokenize/han_solo.py @@ -48,7 +48,12 @@ def _get_tagger() -> pycrfsuite.Tagger: class Featurizer: # This class from ssg at https://github.com/ponrawee/ssg. - def __init__(self, N: int = 2, sequence_size: int = 1, delimiter: Optional[str] = None) -> None: + def __init__( + self, + N: int = 2, + sequence_size: int = 1, + delimiter: Optional[str] = None, + ) -> None: self.N = N self.delimiter = delimiter self.radius = N + sequence_size @@ -57,7 +62,11 @@ def pad(self, sentence: str, padder: str = "#") -> str: return padder * (self.radius) + sentence + padder * (self.radius) def featurize( - self, sentence: str, padding: bool = True, indiv_char: bool = True, return_type: str = "list" + self, + sentence: str, + padding: bool = True, + indiv_char: bool = True, + return_type: str = "list", ) -> dict[str, list]: if padding: sentence = self.pad(sentence) @@ -126,7 +135,7 @@ def featurize( if return_type == "list": return { "X": all_features_list, - "Y": [str(label) for label in all_labels_int] + "Y": [str(label) for label in all_labels_int], } else: return { @@ -134,7 +143,7 @@ def featurize( {key: 1 for key in feature_list} for feature_list in all_features_list ], - "Y": all_labels_int + "Y": all_labels_int, } diff --git a/pythainlp/tokenize/longest.py b/pythainlp/tokenize/longest.py index 1712559be..684544830 100644 --- a/pythainlp/tokenize/longest.py +++ b/pythainlp/tokenize/longest.py @@ -182,7 +182,9 @@ def segment(text: str, custom_dict: Optional[Trie] = None) -> list[str]: # Thread-safe access to the tokenizers cache with _tokenizers_lock: if custom_dict_ref_id not in _tokenizers: - _tokenizers[custom_dict_ref_id] = LongestMatchTokenizer(custom_dict) + _tokenizers[custom_dict_ref_id] = LongestMatchTokenizer( + custom_dict + ) tokenizer = _tokenizers[custom_dict_ref_id] return tokenizer.tokenize(text) diff --git a/pythainlp/tokenize/multi_cut.py b/pythainlp/tokenize/multi_cut.py index 08aba38c5..d6d7d1940 100644 --- a/pythainlp/tokenize/multi_cut.py +++ b/pythainlp/tokenize/multi_cut.py @@ -55,7 +55,9 @@ def _multicut( if not custom_dict: custom_dict = word_dict_trie() len_text = len(text) - words_at: defaultdict[int, list[str]] = defaultdict(list) # main data structure + words_at: defaultdict[int, list[str]] = defaultdict( + list + ) # main data structure def serialize(p, p2): # helper function for w in words_at[p]: @@ -142,7 +144,9 @@ def segment(text: str, custom_dict: Optional[Trie] = None) -> list[str]: return list(_multicut(text, custom_dict=custom_dict)) -def find_all_segment(text: str, custom_dict: Optional[Trie] = None) -> list[str]: +def find_all_segment( + text: str, custom_dict: Optional[Trie] = None +) -> list[str]: """Get all possible segment variations. :param text: input string to be tokenized diff --git a/pythainlp/tokenize/ssg.py b/pythainlp/tokenize/ssg.py index 431a77032..50b929cfe 100644 --- a/pythainlp/tokenize/ssg.py +++ b/pythainlp/tokenize/ssg.py @@ -9,8 +9,7 @@ def segment(text: str) -> list[str]: - """Syllable tokenizer using ssg - """ + """Syllable tokenizer using ssg""" if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 36596032a..bcbb1e166 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -43,11 +43,15 @@ def middle_cut(sentences: list[str]) -> list[str]: continue if sentence[k].isdigit() and sentence[k - 1] == " ": sentence = sentence[: k - 1] + sentence[k:] - sentence_len = len(sentence) # Update length after modification + sentence_len = len( + sentence + ) # Update length after modification if k + 2 <= sentence_len: if sentence[k].isdigit() and sentence[k + 1] == " ": sentence = sentence[: k + 1] + sentence[k + 2 :] - sentence_len = len(sentence) # Update length after modification + sentence_len = len( + sentence + ) # Update length after modification fixed_text_lenth = 20 @@ -64,10 +68,14 @@ def middle_cut(sentences: list[str]) -> list[str]: white_space_index.append(j) for white_space in white_space_index: - white_space_diff[white_space] = abs(white_space - middle_space) + white_space_diff[white_space] = abs( + white_space - middle_space + ) if white_space_diff: - min_diff = min(white_space_diff.items(), key=operator.itemgetter(1)) + min_diff = min( + white_space_diff.items(), key=operator.itemgetter(1) + ) tokens.pop(min_diff[0]) tokens.insert(min_diff[0], "") result_parts.append(list_to_string(tokens)) @@ -75,19 +83,21 @@ def middle_cut(sentences: list[str]) -> list[str]: result_parts.append(sentence) # Split all result parts by and filter - all_sentences = (s.strip() for part in result_parts for s in part.split("")) + all_sentences = ( + s.strip() for part in result_parts for s in part.split("") + ) return list(filter(None, all_sentences)) class ThaiSentenceSegmentor: - def split_into_sentences(self, text: str, isMiddleCut: bool = False) -> list[str]: + def split_into_sentences( + self, text: str, isMiddleCut: bool = False + ) -> list[str]: # Declare Variables th_alphabets = "([ก-๙])" th_conjunction = "(ทำให้|โดย|เพราะ|นอกจากนี้|แต่|กรณีที่|หลังจากนี้|ต่อมา|ภายหลัง|นับตั้งแต่|หลังจาก|ซึ่งเหตุการณ์|ผู้สื่อข่าวรายงานอีก|ส่วนที่|ส่วนสาเหตุ|ฉะนั้น|เพราะฉะนั้น|เพื่อ|เนื่องจาก|จากการสอบสวนทราบว่า|จากกรณี|จากนี้|อย่างไรก็ดี)" - th_cite = ( - "(กล่าวว่า|เปิดเผยว่า|รายงานว่า|ให้การว่า|เผยว่า|บนทวิตเตอร์ว่า|แจ้งว่า|พลเมืองดีว่า|อ้างว่า)" - ) + th_cite = "(กล่าวว่า|เปิดเผยว่า|รายงานว่า|ให้การว่า|เผยว่า|บนทวิตเตอร์ว่า|แจ้งว่า|พลเมืองดีว่า|อ้างว่า)" th_ka_krub = "(ครับ|ค่ะ)" th_stop_after = "(หรือไม่|โดยเร็ว|แล้ว|อีกด้วย)" th_stop_before = "(ล่าสุด|เบื้องต้น|ซึ่ง|ทั้งนี้|แม้ว่า|เมื่อ|แถมยัง|ตอนนั้น|จนเป็นเหตุให้|จากนั้น|อย่างไรก็ตาม|และก็|อย่างใดก็ตาม|เวลานี้|เช่น|กระทั่ง)" @@ -153,7 +163,9 @@ def split_into_sentences(self, text: str, isMiddleCut: bool = False) -> list[str text = text.replace("ทั้งนี้เพื่อ", "ทั้งนี้") text = text.replace("เวลาต่อมา", "เวลา") text = text.replace("อย่างไรก็ตาม", "อย่างไรก็ตาม") - text = text.replace("อย่างไรก็ตามหลังจาก", "อย่างไรก็ตาม") + text = text.replace( + "อย่างไรก็ตามหลังจาก", "อย่างไรก็ตาม" + ) text = text.replace("ซึ่งทำให้", "ซึ่ง") text = text.replace("โดยประมาท", "ประมาท") text = text.replace("โดยธรรม", "ธรรม") @@ -282,7 +294,9 @@ def split_into_sentences(self, text: str, isMiddleCut: bool = False) -> list[str text = re.sub(th_conjunction, "\\1", text) text = re.sub(th_cite, "\\1", text) text = re.sub(" " + degit + "[.]" + th_title, "\\1.\\2", text) - text = re.sub(" " + degit + degit + "[.]" + th_title, "\\1\\2.\\3", text) + text = re.sub( + " " + degit + degit + "[.]" + th_title, "\\1\\2.\\3", text + ) text = re.sub(th_alphabets + th_stop_after + " ", "\\1\\2", text) if "”" in text: text = text.replace(".”", "”.") diff --git a/pythainlp/tools/core.py b/pythainlp/tools/core.py index 53038af1d..e4002cd15 100644 --- a/pythainlp/tools/core.py +++ b/pythainlp/tools/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Generic support functions for PyThaiNLP. -""" +"""Generic support functions for PyThaiNLP.""" from __future__ import annotations diff --git a/pythainlp/tools/misspell.py b/pythainlp/tools/misspell.py index b076592b9..f455b4b07 100644 --- a/pythainlp/tools/misspell.py +++ b/pythainlp/tools/misspell.py @@ -42,7 +42,9 @@ ] -def search_location_of_character(char: str) -> Optional[tuple[int, int, int, int]]: +def search_location_of_character( + char: str, +) -> Optional[tuple[int, int, int, int]]: for language_ix in [0, 1]: for ix, row in enumerate(ALL_CHARACTERS[language_ix]): if char in row: @@ -53,7 +55,14 @@ def search_location_of_character(char: str) -> Optional[tuple[int, int, int, int def find_neighbour_locations( loc: tuple[int, int, int, int], char: str, - kernel: list[tuple[int, int]] = [(-1, -1), (-1, 0), (1, 1), (0, 1), (0, -1), (1, 0)], + kernel: list[tuple[int, int]] = [ + (-1, -1), + (-1, 0), + (1, 1), + (0, 1), + (0, -1), + (1, 0), + ], ) -> list[tuple[int, int, int, int, str]]: language_ix, is_shift, row, pos = loc @@ -68,7 +77,9 @@ def find_neighbour_locations( return valid_neighbours -def find_misspell_candidates(char: str, verbose: bool = False) -> Optional[list[str]]: +def find_misspell_candidates( + char: str, verbose: bool = False +) -> Optional[list[str]]: loc = search_location_of_character(char) if loc is None: return None diff --git a/pythainlp/translate/__init__.py b/pythainlp/translate/__init__.py index f126f56db..30dccd8e5 100644 --- a/pythainlp/translate/__init__.py +++ b/pythainlp/translate/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Language translation. -""" +"""Language translation.""" __all__ = ["Translate", "ThZhTranslator", "ZhThTranslator", "word_translate"] diff --git a/pythainlp/translate/core.py b/pythainlp/translate/core.py index 545f9ff89..02c69606f 100644 --- a/pythainlp/translate/core.py +++ b/pythainlp/translate/core.py @@ -13,8 +13,7 @@ class Translate: - """Machine Translation - """ + """Machine Translation""" def __init__( self, @@ -105,7 +104,7 @@ def translate(self, text: str) -> str: :rtype: str """ if self.engine == "small100": - return self.model.translate(text, tgt_lang=self.target_lang) + return self.model.translate(text, tgt_lang=self.target_lang) # type: ignore[call-arg] return self.model.translate(text) diff --git a/pythainlp/translate/en_th.py b/pythainlp/translate/en_th.py index b68ada170..8b08a491f 100644 --- a/pythainlp/translate/en_th.py +++ b/pythainlp/translate/en_th.py @@ -50,8 +50,7 @@ def _download_install(name: str) -> None: def download_model_all() -> None: - """Download all translation models in advance - """ + """Download all translation models in advance""" _download_install(_EN_TH_MODEL_NAME) _download_install(_TH_EN_MODEL_NAME) diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index 9a4b0ca51..5d67412d3 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -137,12 +137,15 @@ def __init__( num_madeup_words=8, **kwargs, ) -> None: - self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + self.sp_model_kwargs = ( + {} if sp_model_kwargs is None else sp_model_kwargs + ) self.language_codes = language_codes fairseq_language_code = FAIRSEQ_LANGUAGE_CODES[language_codes] self.lang_code_to_token = { - lang_code: f"__{lang_code}__" for lang_code in fairseq_language_code + lang_code: f"__{lang_code}__" + for lang_code in fairseq_language_code } kwargs["additional_special_tokens"] = kwargs.get( @@ -151,7 +154,8 @@ def __init__( kwargs["additional_special_tokens"] += [ self.get_lang_token(lang_code) for lang_code in fairseq_language_code - if self.get_lang_token(lang_code) not in kwargs["additional_special_tokens"] + if self.get_lang_token(lang_code) + not in kwargs["additional_special_tokens"] ] super().__init__( @@ -186,7 +190,9 @@ def __init__( lang_code: self.encoder_size + i for i, lang_code in enumerate(fairseq_language_code) } - self.id_to_lang_token = {v: k for k, v in self.lang_token_to_id.items()} + self.id_to_lang_token = { + v: k for k, v in self.lang_token_to_id.items() + } self._tgt_lang = tgt_lang if tgt_lang is not None else "en" self.cur_lang_id = self.get_lang_id(self._tgt_lang) @@ -196,7 +202,11 @@ def __init__( @property def vocab_size(self) -> int: - return len(self.encoder) + len(self.lang_token_to_id) + self.num_madeup_words # type: ignore[no-any-return] + return ( + len(self.encoder) + + len(self.lang_token_to_id) + + self.num_madeup_words + ) # type: ignore[no-any-return] @property def tgt_lang(self) -> str: @@ -222,7 +232,7 @@ def _convert_id_to_token(self, index: int) -> str: token = self.decoder.get(index, self.unk_token) if token is None: return self.unk_token # type: ignore[no-any-return] - return token # type: ignore[no-any-return] + return token def convert_tokens_to_string(self, tokens: list[str]) -> str: """Converts a sequence of tokens (strings for sub-words) in a single string.""" @@ -260,7 +270,9 @@ def get_special_tokens_mask( already_has_special_tokens=True, ) - prefix_ones = [1] * len(self.prefix_tokens) if self.prefix_tokens else [] + prefix_ones = ( + [1] * len(self.prefix_tokens) if self.prefix_tokens else [] + ) suffix_ones = [1] * len(self.suffix_tokens) if token_ids_1 is None: return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones @@ -305,10 +317,17 @@ def build_inputs_with_special_tokens( if self.prefix_tokens is None: return token_ids_0 + token_ids_1 + self.suffix_tokens else: - return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens + return ( + self.prefix_tokens + + token_ids_0 + + token_ids_1 + + self.suffix_tokens + ) def get_vocab(self) -> dict: - vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} + vocab = { + self.convert_ids_to_tokens(i): i for i in range(self.vocab_size) + } vocab.update(self.added_tokens_encoder) return vocab @@ -371,7 +390,9 @@ def _build_translation_inputs( """Used by translation pipeline, to prepare inputs for the generate function""" if tgt_lang is None: - raise ValueError("Translation requires a `tgt_lang` for this model") + raise ValueError( + "Translation requires a `tgt_lang` for this model" + ) self.tgt_lang = tgt_lang inputs = self(raw_inputs, add_special_tokens=True, **extra_kwargs) return inputs diff --git a/pythainlp/transliterate/__init__.py b/pythainlp/transliterate/__init__.py index bee56ad24..ecd556e05 100644 --- a/pythainlp/transliterate/__init__.py +++ b/pythainlp/transliterate/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Transliteration. -""" +"""Transliteration.""" __all__ = [ "pronunciate", diff --git a/pythainlp/transliterate/lookup.py b/pythainlp/transliterate/lookup.py index 4b0d1964e..58c7333c2 100644 --- a/pythainlp/transliterate/lookup.py +++ b/pythainlp/transliterate/lookup.py @@ -40,8 +40,7 @@ def follow_rtgs(text: str) -> Optional[bool]: def _romanize(text: str, fallback_func: Callable[[str], str]) -> str: - """Romanize one word. Look up first, call `fallback_func` if not found. - """ + """Romanize one word. Look up first, call `fallback_func` if not found.""" try: # try to get 0-th idx of look up result, simply ignore other possible variations. # not found means no mapping. diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index faf268f26..155814b9f 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -43,7 +43,9 @@ def __init__(self): # Restore the model and construct the encoder and decoder. self._encoder = Encoder(INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT) - self._decoder = AttentionDecoder(OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT) + self._decoder = AttentionDecoder( + OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT + ) self._network = Seq2Seq( self._encoder, @@ -75,7 +77,9 @@ def romanize(self, text: str) -> str: """ input_tensor = self._prepare_sequence_in(text).view(1, -1) input_length = torch.Tensor([len(text) + 1]).int() - target_tensor_logits = self._network(input_tensor, input_length, None, 0) + target_tensor_logits = self._network( + input_tensor, input_length, None, 0 + ) # Seq2seq model returns as the first token, # As a result, target_tensor_logits.size() is torch.Size([0]) @@ -83,7 +87,10 @@ def romanize(self, text: str) -> str: target = [""] else: target_tensor = ( - torch.argmax(target_tensor_logits.squeeze(1), 1).cpu().detach().numpy() + torch.argmax(target_tensor_logits.squeeze(1), 1) + .cpu() + .detach() + .numpy() ) target = [self._ix_to_target_char[t] for t in target_tensor] @@ -91,11 +98,15 @@ def romanize(self, text: str) -> str: class Encoder(nn.Module): - def __init__(self, vocabulary_size, embedding_size, hidden_size, dropout=0.5): + def __init__( + self, vocabulary_size, embedding_size, hidden_size, dropout=0.5 + ): """Constructor""" super().__init__() self.hidden_size = hidden_size - self.character_embedding = nn.Embedding(vocabulary_size, embedding_size) + self.character_embedding = nn.Embedding( + vocabulary_size, embedding_size + ) self.rnn = nn.LSTM( input_size=embedding_size, hidden_size=hidden_size // 2, @@ -167,9 +178,9 @@ def __init__(self, method, hidden_size): def forward(self, hidden, encoder_outputs, mask): # Calculate energies for each encoder output if self.method == "dot": - attn_energies = torch.bmm(encoder_outputs, hidden.transpose(1, 2)).squeeze( - 2 - ) + attn_energies = torch.bmm( + encoder_outputs, hidden.transpose(1, 2) + ).squeeze(2) elif self.method == "general": attn_energies = self.attn( encoder_outputs.view(-1, encoder_outputs.size(-1)) @@ -177,9 +188,7 @@ def forward(self, hidden, encoder_outputs, mask): attn_energies = torch.bmm( attn_energies.view(*encoder_outputs.size()), hidden.transpose(1, 2), - ).squeeze( - 2 - ) # (batch_size, sequence_len) + ).squeeze(2) # (batch_size, sequence_len) elif self.method == "concat": attn_energies = self.attn( torch.cat( @@ -199,12 +208,16 @@ def forward(self, hidden, encoder_outputs, mask): class AttentionDecoder(nn.Module): - def __init__(self, vocabulary_size, embedding_size, hidden_size, dropout=0.5): + def __init__( + self, vocabulary_size, embedding_size, hidden_size, dropout=0.5 + ): """Constructor""" super().__init__() self.vocabulary_size = vocabulary_size self.hidden_size = hidden_size - self.character_embedding = nn.Embedding(vocabulary_size, embedding_size) + self.character_embedding = nn.Embedding( + vocabulary_size, embedding_size + ) self.rnn = nn.LSTM( input_size=embedding_size + self.hidden_size, hidden_size=hidden_size, @@ -281,7 +294,9 @@ def forward( max_len = self.max_length target_vocab_size = self.decoder.vocabulary_size - outputs = torch.zeros(max_len, batch_size, target_vocab_size).to(device) + outputs = torch.zeros(max_len, batch_size, target_vocab_size).to( + device + ) if target_seq is None: assert teacher_forcing_ratio == 0, "Must be zero during inference" @@ -289,10 +304,14 @@ def forward( else: inference = False - encoder_outputs, encoder_hidden = self.encoder(source_seq, source_seq_len) + encoder_outputs, encoder_hidden = self.encoder( + source_seq, source_seq_len + ) decoder_input = ( - torch.tensor([[start_token] * batch_size]).view(batch_size, 1).to(device) + torch.tensor([[start_token] * batch_size]) + .view(batch_size, 1) + .to(device) ) encoder_hidden_h_t = torch.cat( diff --git a/pythainlp/transliterate/thai2rom_onnx.py b/pythainlp/transliterate/thai2rom_onnx.py index ab00131d2..e986afb9c 100644 --- a/pythainlp/transliterate/thai2rom_onnx.py +++ b/pythainlp/transliterate/thai2rom_onnx.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Romanization of Thai words based on machine-learnt engine in ONNX runtime ("thai2rom") -""" +"""Romanization of Thai words based on machine-learnt engine in ONNX runtime ("thai2rom")""" from __future__ import annotations @@ -58,8 +57,7 @@ def __init__(self): ) def _prepare_sequence_in(self, text: str): - """Prepare input sequence for ONNX - """ + """Prepare input sequence for ONNX""" idxs = [] for ch in text: if ch in self._char_to_ix: diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index be867b80f..eb8614512 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -53,7 +53,9 @@ def __init__(self): # Restore the model and construct the encoder and decoder. self._encoder = Encoder(INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT) - self._decoder = AttentionDecoder(OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT) + self._decoder = AttentionDecoder( + OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT + ) self._network = Seq2Seq( self._encoder, @@ -86,7 +88,9 @@ def g2p(self, text: str) -> str: input_tensor = self._prepare_sequence_in(text).view(1, -1) input_length = [len(text) + 1] - target_tensor_logits = self._network(input_tensor, input_length, None, 0) + target_tensor_logits = self._network( + input_tensor, input_length, None, 0 + ) # Seq2seq model returns as the first token, # As a result, target_tensor_logits.size() is torch.Size([0]) @@ -94,7 +98,10 @@ def g2p(self, text: str) -> str: target = [""] else: target_tensor = ( - torch.argmax(target_tensor_logits.squeeze(1), 1).cpu().detach().numpy() + torch.argmax(target_tensor_logits.squeeze(1), 1) + .cpu() + .detach() + .numpy() ) target = [self._ix_to_target_char[t] for t in target_tensor] @@ -102,11 +109,15 @@ def g2p(self, text: str) -> str: class Encoder(nn.Module): - def __init__(self, vocabulary_size, embedding_size, hidden_size, dropout=0.5): + def __init__( + self, vocabulary_size, embedding_size, hidden_size, dropout=0.5 + ): """Constructor""" super().__init__() self.hidden_size = hidden_size - self.character_embedding = nn.Embedding(vocabulary_size, embedding_size) + self.character_embedding = nn.Embedding( + vocabulary_size, embedding_size + ) self.rnn = nn.LSTM( input_size=embedding_size, hidden_size=hidden_size // 2, @@ -180,9 +191,9 @@ def __init__(self, method, hidden_size): def forward(self, hidden, encoder_outputs, mask): # Calculate energies for each encoder output if self.method == "dot": - attn_energies = torch.bmm(encoder_outputs, hidden.transpose(1, 2)).squeeze( - 2 - ) + attn_energies = torch.bmm( + encoder_outputs, hidden.transpose(1, 2) + ).squeeze(2) elif self.method == "general": attn_energies = self.attn( encoder_outputs.view(-1, encoder_outputs.size(-1)) @@ -190,9 +201,7 @@ def forward(self, hidden, encoder_outputs, mask): attn_energies = torch.bmm( attn_energies.view(*encoder_outputs.size()), hidden.transpose(1, 2), - ).squeeze( - 2 - ) # (batch_size, sequence_len) + ).squeeze(2) # (batch_size, sequence_len) elif self.method == "concat": attn_energies = self.attn( torch.cat( @@ -212,12 +221,16 @@ def forward(self, hidden, encoder_outputs, mask): class AttentionDecoder(nn.Module): - def __init__(self, vocabulary_size, embedding_size, hidden_size, dropout=0.5): + def __init__( + self, vocabulary_size, embedding_size, hidden_size, dropout=0.5 + ): """Constructor""" super().__init__() self.vocabulary_size = vocabulary_size self.hidden_size = hidden_size - self.character_embedding = nn.Embedding(vocabulary_size, embedding_size) + self.character_embedding = nn.Embedding( + vocabulary_size, embedding_size + ) self.rnn = nn.LSTM( input_size=embedding_size + self.hidden_size, hidden_size=hidden_size, @@ -294,7 +307,9 @@ def forward( max_len = self.max_length target_vocab_size = self.decoder.vocabulary_size - outputs = torch.zeros(max_len, batch_size, target_vocab_size).to(device) + outputs = torch.zeros(max_len, batch_size, target_vocab_size).to( + device + ) if target_seq is None: assert teacher_forcing_ratio == 0, "Must be zero during inference" @@ -302,10 +317,14 @@ def forward( else: inference = False - encoder_outputs, encoder_hidden = self.encoder(source_seq, source_seq_len) + encoder_outputs, encoder_hidden = self.encoder( + source_seq, source_seq_len + ) decoder_input = ( - torch.tensor([[start_token] * batch_size]).view(batch_size, 1).to(device) + torch.tensor([[start_token] * batch_size]) + .view(batch_size, 1) + .to(device) ) encoder_hidden_h_t = torch.cat( diff --git a/pythainlp/transliterate/w2p.py b/pythainlp/transliterate/w2p.py index 30d031559..24a153080 100644 --- a/pythainlp/transliterate/w2p.py +++ b/pythainlp/transliterate/w2p.py @@ -122,9 +122,9 @@ def _gru(self, x, steps, w_ih, w_hh, b_ih, b_hh, h0=None) -> np.ndarray: h = self._grucell(x[:, t, :], h, w_ih, w_hh, b_ih, b_hh) # (b, h) outputs[:, t, ::] = h - return outputs + return outputs # type: ignore[no-any-return] - def _encode(self, word: str) -> np.ndarray: # type: ignore[type-arg] + def _encode(self, word: str) -> np.ndarray: chars = list(word) + [""] x = [self.g2idx.get(char, self.g2idx[""]) for char in chars] x = np.take(self.enc_emb, np.expand_dims(x, 0), axis=0) diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index 8da11115a..7c3a98519 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Universal Language Model Fine-tuning for Text Classification (ULMFiT). -""" +"""Universal Language Model Fine-tuning for Text Classification (ULMFiT).""" from __future__ import annotations @@ -71,6 +70,7 @@ def get_thwiki_lstm() -> dict[str, str]: return {"wgts_fname": wgts_fname, "itos_fname": itos_fname} + # Preprocessing rules for Thai text # dense features pre_rules_th = [ @@ -169,7 +169,7 @@ def process_thai( for rule in post_rules: res = rule(res) - return res + return res # type: ignore[no-any-return] def document_vector(text: str, learn, data, agg: str = "mean"): diff --git a/pythainlp/ulmfit/preprocess.py b/pythainlp/ulmfit/preprocess.py index 346cfffa3..0c73dfdcf 100644 --- a/pythainlp/ulmfit/preprocess.py +++ b/pythainlp/ulmfit/preprocess.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Preprocessing for ULMFiT -""" +"""Preprocessing for ULMFiT""" from __future__ import annotations diff --git a/pythainlp/ulmfit/tokenizer.py b/pythainlp/ulmfit/tokenizer.py index 2ce042773..00810e8a8 100644 --- a/pythainlp/ulmfit/tokenizer.py +++ b/pythainlp/ulmfit/tokenizer.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Tokenzier classes for ULMFiT -""" +"""Tokenzier classes for ULMFiT""" from __future__ import annotations @@ -63,7 +62,7 @@ def tokenizer(text: str) -> list[str]: ' ', 'ภาวนามยปัญญา'] """ - return thai2fit_tokenizer().word_tokenize(text) + return thai2fit_tokenizer().word_tokenize(text) # type: ignore[no-any-return] def add_special_cases(self, toks): pass diff --git a/pythainlp/util/abbreviation.py b/pythainlp/util/abbreviation.py index 437707f64..19e080371 100644 --- a/pythainlp/util/abbreviation.py +++ b/pythainlp/util/abbreviation.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Thai abbreviation tools -""" +"""Thai abbreviation tools""" from __future__ import annotations diff --git a/pythainlp/util/date.py b/pythainlp/util/date.py index f0a44bd46..cee0521b3 100644 --- a/pythainlp/util/date.py +++ b/pythainlp/util/date.py @@ -88,8 +88,7 @@ dates_list = ( "(" + "|".join( - list(map(str, range(32, 0, -1))) - + ["0" + str(i) for i in range(1, 10)] + list(map(str, range(32, 0, -1))) + ["0" + str(i) for i in range(1, 10)] ) + ")" ) @@ -380,7 +379,9 @@ def reign_year_to_ad(reign_year: int, reign: int) -> int: return ad -def thaiword_to_date(text: str, date: Optional[datetime] = None) -> Optional[datetime]: +def thaiword_to_date( + text: str, date: Optional[datetime] = None +) -> Optional[datetime]: """Convert Thai relative date to :class:`datetime.datetime`. :param str text: Thai text containing relative date diff --git a/pythainlp/util/digitconv.py b/pythainlp/util/digitconv.py index b80164b04..21cd95029 100644 --- a/pythainlp/util/digitconv.py +++ b/pythainlp/util/digitconv.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Convert digits -""" +"""Convert digits""" from __future__ import annotations diff --git a/pythainlp/util/emojiconv.py b/pythainlp/util/emojiconv.py index 6aa9265b4..eb3121820 100644 --- a/pythainlp/util/emojiconv.py +++ b/pythainlp/util/emojiconv.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Convert emojis -""" +"""Convert emojis""" from __future__ import annotations diff --git a/pythainlp/util/keyboard.py b/pythainlp/util/keyboard.py index 8ae9c2408..f9e99ee49 100644 --- a/pythainlp/util/keyboard.py +++ b/pythainlp/util/keyboard.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Functions related to keyboard layout. -""" +"""Functions related to keyboard layout.""" from __future__ import annotations diff --git a/pythainlp/util/keywords.py b/pythainlp/util/keywords.py index 751bca013..5efc31313 100644 --- a/pythainlp/util/keywords.py +++ b/pythainlp/util/keywords.py @@ -11,7 +11,9 @@ _STOPWORDS = thai_stopwords() -def rank(words: list[str], exclude_stopwords: bool = False) -> Optional[Counter]: +def rank( + words: list[str], exclude_stopwords: bool = False +) -> Optional[Counter]: """Count word frequencies given a list of Thai words with an option to exclude stopwords. diff --git a/pythainlp/util/normalize.py b/pythainlp/util/normalize.py index beb3f4eac..16c527d65 100644 --- a/pythainlp/util/normalize.py +++ b/pythainlp/util/normalize.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Text normalization -""" +"""Text normalization""" from __future__ import annotations diff --git a/pythainlp/util/phoneme.py b/pythainlp/util/phoneme.py index 186f11d8a..dbd0491e5 100644 --- a/pythainlp/util/phoneme.py +++ b/pythainlp/util/phoneme.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Phonemes util -""" +"""Phonemes util""" from __future__ import annotations diff --git a/pythainlp/util/profanity.py b/pythainlp/util/profanity.py index 1ca4d268e..b3b27ce60 100644 --- a/pythainlp/util/profanity.py +++ b/pythainlp/util/profanity.py @@ -4,6 +4,7 @@ """ Profanity detection for Thai language """ + from __future__ import annotations from typing import Optional diff --git a/pythainlp/util/remove_trailing_repeat_consonants.py b/pythainlp/util/remove_trailing_repeat_consonants.py index af1deef5f..17b15cd21 100644 --- a/pythainlp/util/remove_trailing_repeat_consonants.py +++ b/pythainlp/util/remove_trailing_repeat_consonants.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Removement of repeated consonants at the end of words -""" +"""Removement of repeated consonants at the end of words""" from __future__ import annotations diff --git a/pythainlp/util/strftime.py b/pythainlp/util/strftime.py index fa6d6dae4..5b6c87d13 100644 --- a/pythainlp/util/strftime.py +++ b/pythainlp/util/strftime.py @@ -106,7 +106,9 @@ def _thai_strftime(dt_obj: datetime, fmt_char: str) -> str: elif fmt_char == "g": # Same year as in ``%G'', # but as a decimal number without century (00-99). - str_ = (str(int(dt_obj.strftime("%G")) + _BE_AD_DIFFERENCE)[-2:]).zfill(2) + str_ = ( + str(int(dt_obj.strftime("%G")) + _BE_AD_DIFFERENCE)[-2:] + ).zfill(2) elif fmt_char == "v": # BSD extension, ' 6-Oct-1976' str_ = f"{dt_obj.day:>2}-{thai_abbr_months[dt_obj.month - 1]}-{str(dt_obj.year + _BE_AD_DIFFERENCE).zfill(4)}" diff --git a/pythainlp/util/syllable.py b/pythainlp/util/syllable.py index 46610a927..6e90cdf91 100644 --- a/pythainlp/util/syllable.py +++ b/pythainlp/util/syllable.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Syllable tools -""" +"""Syllable tools""" from __future__ import annotations @@ -299,34 +298,32 @@ def tone_detector(syllable: str) -> str: consonant_ending = _check_sonorant_syllable(syllable) if consonant_ending: # Only apply special rules if there are sonorants - if ( - initial_consonant == "อ" - and s == "live" - and tone_mark == "่" - ): + if initial_consonant == "อ" and s == "live" and tone_mark == "่": r = "l" - elif ( - initial_consonant == "ห" - and s == "live" - and tone_mark == "่" - ): + elif initial_consonant == "ห" and s == "live" and tone_mark == "่": r = "l" elif initial_consonant == "อ" and s == "dead": r = "l" - elif ( - initial_consonant == "ห" - and s == "live" - and tone_mark == "้" - ): + elif initial_consonant == "ห" and s == "live" and tone_mark == "้": r = "f" elif initial_consonant == "ห" and s == "dead": r = "l" elif initial_consonant == "ห" and s == "live": r = "r" # If r is still empty, apply general tone rules - if r == "" and initial_consonant_type == "high" and s == "live" and tone_mark == "่": + if ( + r == "" + and initial_consonant_type == "high" + and s == "live" + and tone_mark == "่" + ): r = "l" - if r == "" and initial_consonant_type == "mid" and s == "live" and tone_mark == "่": + if ( + r == "" + and initial_consonant_type == "mid" + and s == "live" + and tone_mark == "่" + ): r = "l" if r == "" and initial_consonant_type == "low" and tone_mark == "้": r = "h" diff --git a/pythainlp/util/thai_lunar_date.py b/pythainlp/util/thai_lunar_date.py index bc8d64382..b5bf3ef19 100644 --- a/pythainlp/util/thai_lunar_date.py +++ b/pythainlp/util/thai_lunar_date.py @@ -281,8 +281,7 @@ def last_day_in_year(year: int) -> int: def athikasurathin(year: int) -> bool: - """Check if a year is a leap year in the Thai lunar calendar - """ + """Check if a year is a leap year in the Thai lunar calendar""" # Check divisibility by 400 (divisible by 400 is always a leap year) if year % 400 == 0: return True diff --git a/pythainlp/util/trie.py b/pythainlp/util/trie.py index d4e3d4b1e..6863291ef 100644 --- a/pythainlp/util/trie.py +++ b/pythainlp/util/trie.py @@ -28,23 +28,24 @@ class Trie(Iterable[str]): from pythainlp.util import Trie # Create a trie with Thai words - trie = Trie(['สวัสดี', 'สวัส', 'ดี', 'ครับ']) + trie = Trie(["สวัสดี", "สวัส", "ดี", "ครับ"]) # Check if word exists - 'สวัสดี' in trie + "สวัสดี" in trie # output: True # Find all prefixes of a word - trie.prefixes('สวัสดีครับ') + trie.prefixes("สวัสดีครับ") # output: ['สวัส', 'สวัสดี'] # Add a new word - trie.add('สวัสดีตอนเช้า') + trie.add("สวัสดีตอนเช้า") # Get number of words in trie len(trie) # output: 5 """ + class Node: __slots__ = "end", "children" diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 11d8ff2b7..7ad0dd0c8 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -68,9 +68,7 @@ def _check_is_thainum(word: str) -> tuple[bool, Optional[str]]: @lru_cache def _tokenizer_thaiwords() -> Tokenizer: """Lazy load Thai words tokenizer with cache""" - _dict_words = [ - i for i in thai_words() if not _check_is_thainum(i)[0] - ] + _dict_words = [i for i in thai_words() if not _check_is_thainum(i)[0]] _dict_words.extend(_digits.keys()) _dict_words.extend(["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"]) return Tokenizer(_dict_words) diff --git a/pythainlp/wangchanberta/core.py b/pythainlp/wangchanberta/core.py index 908d5a49c..526bb55e1 100644 --- a/pythainlp/wangchanberta/core.py +++ b/pythainlp/wangchanberta/core.py @@ -23,12 +23,18 @@ def _get_tokenizer(): f"airesearch/{_model_name}", revision="main" ) if _model_name == "wangchanberta-base-att-spm-uncased": - _tokenizer.additional_special_tokens = ["NOTUSED", "NOTUSED", "<_>"] + _tokenizer.additional_special_tokens = [ + "NOTUSED", + "NOTUSED", + "<_>", + ] return _tokenizer class ThaiNameTagger: - def __init__(self, dataset_name: str = "thainer", grouped_entities: bool = True): + def __init__( + self, dataset_name: str = "thainer", grouped_entities: bool = True + ): """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ @@ -109,7 +115,9 @@ def get_ner( self.sent_ner = self.sent_ner[1:] for idx, (word, ner) in enumerate(self.sent_ner): if idx > 0 and ner.startswith("B-"): - if self._clear_tag(ner) == self._clear_tag(self.sent_ner[idx - 1][1]): + if self._clear_tag(ner) == self._clear_tag( + self.sent_ner[idx - 1][1] + ): self.sent_ner[idx] = (word, ner.replace("B-", "I-")) if tag: temp = "" @@ -136,7 +144,9 @@ def get_ner( class NamedEntityRecognition: - def __init__(self, model: str = "pythainlp/thainer-corpus-v2-base-model") -> None: + def __init__( + self, model: str = "pythainlp/thainer-corpus-v2-base-model" + ) -> None: """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ @@ -199,7 +209,9 @@ def get_ner( predicted_token_class = [ self.model.config.id2label[t.item()] for t in predictions[0] ] - ner_tag = self._fix_span_error(inputs["input_ids"][0], predicted_token_class) + ner_tag = self._fix_span_error( + inputs["input_ids"][0], predicted_token_class + ) if tag: temp = "" sent = "" @@ -220,7 +232,7 @@ def get_ner( sent += "" return sent - return ner_tag + return ner_tag # type: ignore[no-any-return] def segment(text: str) -> list[str]: @@ -233,4 +245,4 @@ def segment(text: str) -> list[str]: if not text or not isinstance(text, str): return [] - return _get_tokenizer().tokenize(text) + return _get_tokenizer().tokenize(text) # type: ignore[no-any-return] diff --git a/pythainlp/wsd/__init__.py b/pythainlp/wsd/__init__.py index 1fce7736d..74a96bfb9 100644 --- a/pythainlp/wsd/__init__.py +++ b/pythainlp/wsd/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Thai Word Sense Disambiguation (WSD) -""" +"""Thai Word Sense Disambiguation (WSD)""" __all__ = ["get_sense"] diff --git a/pythainlp/wsd/core.py b/pythainlp/wsd/core.py index 68d132790..86870d67e 100644 --- a/pythainlp/wsd/core.py +++ b/pythainlp/wsd/core.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from typing import Optional +from typing import Optional, cast from pythainlp.corpus import thai_wsd_dict from pythainlp.tokenize import Tokenizer @@ -15,7 +15,7 @@ for i, j in zip(_wsd_dict["word"], _wsd_dict["meaning"]): _mean_all[i] = j -_all_word = set(_mean_all.keys()) +_all_word = cast(set[str], set(_mean_all.keys())) _TRIE = Trie(_all_word) _word_cut = Tokenizer(custom_dict=_TRIE) @@ -45,7 +45,7 @@ def get_score(self, sentences1: str, sentences2: str) -> float: embedding_1 = self.model.encode(sentences1, convert_to_tensor=True) embedding_2 = self.model.encode(sentences2, convert_to_tensor=True) - return 1 - util.pytorch_cos_sim(embedding_1, embedding_2)[0][0].item() + return 1 - util.pytorch_cos_sim(embedding_1, embedding_2)[0][0].item() # type: ignore[no-any-return] def get_sense( From 964f20923befbe860c8eea7bcd7bb98f753397a4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 3 Feb 2026 11:45:50 +0000 Subject: [PATCH 7/7] Address code review feedback - Fixed wordnet.py to use the tokenize parameter as intended - Fixed thai2fit and ltw2v to properly handle None corpus paths with clear error messages - Fixed tokenization_small100.py type ignore comment formatting - All mypy checks still passing (0 errors) Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/augment/word2vec/ltw2v.py | 8 ++++++-- pythainlp/augment/word2vec/thai2fit.py | 8 ++++++-- pythainlp/augment/wordnet.py | 2 +- pythainlp/translate/tokenization_small100.py | 7 ++----- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pythainlp/augment/word2vec/ltw2v.py b/pythainlp/augment/word2vec/ltw2v.py index 4be52f759..34d1a8de2 100644 --- a/pythainlp/augment/word2vec/ltw2v.py +++ b/pythainlp/augment/word2vec/ltw2v.py @@ -27,8 +27,12 @@ def tokenizer(self, text: str) -> list[str]: def load_w2v(self): # insert substitute """Load LTW2V's word2vec model""" - ltw2v_wv = self.ltw2v_wv or "" - self.aug = Word2VecAug(ltw2v_wv, self.tokenizer, type="binary") + if self.ltw2v_wv is None: + raise ValueError( + "LTW2V word2vec model not found. " + "Please download it first using pythainlp.corpus.download('ltw2v_wv')" + ) + self.aug = Word2VecAug(self.ltw2v_wv, self.tokenizer, type="binary") def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 diff --git a/pythainlp/augment/word2vec/thai2fit.py b/pythainlp/augment/word2vec/thai2fit.py index cb4a810d4..d24ae10a2 100644 --- a/pythainlp/augment/word2vec/thai2fit.py +++ b/pythainlp/augment/word2vec/thai2fit.py @@ -28,8 +28,12 @@ def tokenizer(self, text: str) -> list[str]: def load_w2v(self): """Load Thai2Fit's word2vec model""" - thai2fit_wv = self.thai2fit_wv or "" - self.aug = Word2VecAug(thai2fit_wv, self.tokenizer, type="binary") + if self.thai2fit_wv is None: + raise ValueError( + "Thai2Fit word2vec model not found. " + "Please download it first using pythainlp.corpus.download('thai2fit_wv')" + ) + self.aug = Word2VecAug(self.thai2fit_wv, self.tokenizer, type="binary") def augment( self, sentence: str, n_sent: int = 1, p: float = 0.7 diff --git a/pythainlp/augment/wordnet.py b/pythainlp/augment/wordnet.py index 8ce64ed1d..80eb3aefe 100644 --- a/pythainlp/augment/wordnet.py +++ b/pythainlp/augment/wordnet.py @@ -188,7 +188,7 @@ def augment( ('เรา', 'ชอบ', 'ไปยัง', 'รร.')] """ new_sentences = [] - self.list_words = word_tokenize(sentence) + self.list_words = tokenize(sentence) self.list_synonym = [] self.p_all = 1 if postag: diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index 5d67412d3..48623ed9d 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -202,11 +202,8 @@ def __init__( @property def vocab_size(self) -> int: - return ( - len(self.encoder) - + len(self.lang_token_to_id) - + self.num_madeup_words - ) # type: ignore[no-any-return] + # Type ignore for external library dict operations + return len(self.encoder) + len(self.lang_token_to_id) + self.num_madeup_words # type: ignore[no-any-return] @property def tgt_lang(self) -> str: