From f790a8ab5aea2ebf3bc346a8208622f9a7b84a4e Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Mon, 12 Jan 2026 12:55:18 +0000 Subject: [PATCH] Add more lint rules and fix some - Add more lint rules - Fix easy lint rule violations - Ignore few for future fixes --- .github/workflows/lint.yml | 2 +- pyproject.toml | 30 ++++++++-- pythainlp/augment/lm/phayathaibert.py | 23 +++----- pythainlp/cli/tokenize.py | 11 ++-- pythainlp/corpus/__init__.py | 14 ++--- pythainlp/corpus/core.py | 3 +- pythainlp/generate/core.py | 24 ++++---- pythainlp/generate/thai2fit.py | 3 +- pythainlp/generate/wangchanglm.py | 2 +- pythainlp/phayathaibert/core.py | 51 +++++++---------- pythainlp/tokenize/thaisumcut.py | 22 +++---- pythainlp/tools/misspell.py | 3 +- pythainlp/translate/tokenization_small100.py | 10 ++-- pythainlp/translate/zh_th.py | 2 + pythainlp/transliterate/thai2rom.py | 60 +++++++------------- pythainlp/transliterate/thaig2p.py | 60 +++++++------------- pythainlp/util/__init__.py | 9 ++- pythainlp/util/strftime.py | 13 ++--- pythainlp/wangchanberta/core.py | 22 +++---- 19 files changed, 160 insertions(+), 204 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7a628b584..0849cfcb0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,4 +26,4 @@ jobs: uses: astral-sh/ruff-action@v3 with: src: "./pythainlp ./tests ./examples" - args: check --fix --verbose --line-length 79 --select I,W,C901,W291,W293 + args: check --verbose --config pyproject.toml diff --git a/pyproject.toml b/pyproject.toml index 6a6f6f6da..bc3fb0d08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -287,18 +287,36 @@ source = ["pythainlp"] # Ruff configuration [tool.ruff] -line-length = 79 indent-width = 4 +line-length = 79 target-version = "py39" [tool.ruff.format] -quote-style = "double" +docstring-code-format = true indent-style = "space" -skip-magic-trailing-comma = false line-ending = "auto" -docstring-code-format = true +quote-style = "double" +skip-magic-trailing-comma = false -[tool.ruff.lint.mccabe] +[tool.ruff.lint] # Flag errors (`C901`) whenever the complexity level exceeds 5. Default is 10. # We should aim to gradually reduce this to 10. -max-complexity = 38 +mccabe.max-complexity = 38 +select = [ + "E", + "F", + "I", + "W", + "S", + "B028", + "C901", + "F401", + "F811", + "F841", + "W291", + "W293", +] +# Some rules are ignored for now; we can consider enabling them in the future. +# F403 and F405 are ignored due to thai2fit/ulmfit module's star imports. +# S101 is use of assert statement, should be an easy fix. +extend-ignore = ["E402", "E501", "E722", "F403", "F405", "S101", "S202", "S301", "S310"] diff --git a/pythainlp/augment/lm/phayathaibert.py b/pythainlp/augment/lm/phayathaibert.py index a8cf8b4a0..c9dceb85e 100644 --- a/pythainlp/augment/lm/phayathaibert.py +++ b/pythainlp/augment/lm/phayathaibert.py @@ -20,9 +20,7 @@ def __init__(self) -> None: ) self.tokenizer = AutoTokenizer.from_pretrained(_MODEL_NAME) - self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained( - _MODEL_NAME - ) + self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained(_MODEL_NAME) self.model = pipeline( "fill-mask", tokenizer=self.tokenizer, @@ -40,13 +38,14 @@ def generate( sample_txt = sample_text final_text = "" - for j in range(max_length): - input = self.processor.preprocess(sample_txt) + for _ in range(max_length): + input_text = self.processor.preprocess(sample_txt) if sample: - random_word_idx = random.randint(0, 4) - output = self.model(input)[random_word_idx]["sequence"] + # Non-cryptographic use, pseudo-random generator is acceptable here + random_word_idx = random.randint(0, 4) # noqa: S311 + output = self.model(input_text)[random_word_idx]["sequence"] else: - output = self.model(input)[word_rank]["sequence"] + output = self.model(input_text)[word_rank]["sequence"] sample_txt = output + "" final_text = sample_txt @@ -54,9 +53,7 @@ def generate( return gen_txt - def augment( - self, text: str, num_augs: int = 3, sample: bool = False - ) -> list[str]: + def augment(self, text: str, num_augs: int = 3, sample: bool = False) -> list[str]: """Text augmentation from PhayaThaiBERT :param str text: Thai text @@ -90,9 +87,7 @@ def augment( if num_augs <= MAX_NUM_AUGS: for rank in range(num_augs): gen_text = self.generate(text, rank, sample=sample) - processed_text = re.sub( - "<_>", " ", self.processor.preprocess(gen_text) - ) + processed_text = re.sub("<_>", " ", self.processor.preprocess(gen_text)) augment_list.append(processed_text) else: raise ValueError( diff --git a/pythainlp/cli/tokenize.py b/pythainlp/cli/tokenize.py index 066414f53..723689896 100644 --- a/pythainlp/cli/tokenize.py +++ b/pythainlp/cli/tokenize.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Command line for PyThaiNLP's tokenizers. -""" +"""Command line for PyThaiNLP's tokenizers.""" import argparse @@ -17,10 +16,10 @@ ) from pythainlp.tools import safe_print -DEFAULT_SENT_TOKEN_SEPARATOR = "@@" -DEFAULT_SUBWORD_TOKEN_SEPARATOR = "/" -DEFAULT_SYLLABLE_TOKEN_SEPARATOR = "~" -DEFAULT_WORD_TOKEN_SEPARATOR = "|" +DEFAULT_SENT_TOKEN_SEPARATOR = "@@" # noqa: S105 +DEFAULT_SUBWORD_TOKEN_SEPARATOR = "/" # noqa: S105 +DEFAULT_SYLLABLE_TOKEN_SEPARATOR = "~" # noqa: S105 +DEFAULT_WORD_TOKEN_SEPARATOR = "|" # noqa: S105 class SubAppBase: diff --git a/pythainlp/corpus/__init__.py b/pythainlp/corpus/__init__.py index 6d04c4593..4b532d710 100644 --- a/pythainlp/corpus/__init__.py +++ b/pythainlp/corpus/__init__.py @@ -71,25 +71,23 @@ def corpus_path() -> str: - """Get path where corpus files are kept locally. - """ + """Get path where corpus files are kept locally.""" return _CORPUS_PATH def corpus_db_url() -> str: - """Get remote URL of corpus catalog. - """ + """Get remote URL of corpus catalog.""" return _CORPUS_DB_URL def corpus_db_path() -> str: - """Get local path of corpus catalog. - """ + """Get local path of corpus catalog.""" return _CORPUS_DB_PATH -# DO NOT REORDER these pythainlp.corpus imports. + +# DO NOT REORDER these pythainlp.corpus.core imports. # These imports must come before other pythainlp.corpus.* imports -from pythainlp.corpus.core import ( +from pythainlp.corpus.core import ( # noqa: I001 download, get_corpus, get_corpus_as_is, diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index 689c28bd0..e27af19f7 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -322,7 +322,8 @@ def _check_hash(dst: str, md5: str) -> None: with open(get_full_data_path(dst), "rb") as f: content = f.read() - file_md5 = hashlib.md5(content).hexdigest() + # MD5 is insecure but sufficient here + file_md5 = hashlib.md5(content).hexdigest() # noqa: S324 if md5 != file_md5: raise ValueError("Hash does not match expected.") diff --git a/pythainlp/generate/core.py b/pythainlp/generate/core.py index bf317e6a7..dc1070d7f 100644 --- a/pythainlp/generate/core.py +++ b/pythainlp/generate/core.py @@ -70,16 +70,15 @@ def gen_sentence( # output: 'แมวเวลานะนั้น' """ if not start_seq: - start_seq = random.choice(self.word) + # Non-cryptographic use, pseudo-random generator is acceptable here + start_seq = random.choice(self.word) # noqa: S311 rand_text = start_seq.lower() self._word_prob = { i: self.counts[i] / self.n for i in self.word if self.counts[i] / self.n >= prob } - return self._next_word( - rand_text, N, output_str, prob=prob, duplicate=duplicate - ) + return self._next_word(rand_text, N, output_str, prob=prob, duplicate=duplicate) def _next_word( self, @@ -95,10 +94,11 @@ def _next_word( if N > len(word_list): N = len(word_list) for _ in range(N): - w = random.choice(word_list) + # Non-cryptographic use, pseudo-random generator is acceptable here + w = random.choice(word_list) # noqa: S311 if duplicate is False: while w in words: - w = random.choice(word_list) + w = random.choice(word_list) # noqa: S311 words.append(w) if output_str: @@ -163,7 +163,8 @@ def gen_sentence( # output: 'แมวไม่ได้รับเชื้อมัน' """ if not start_seq: - start_seq = random.choice(self.words) + # Non-cryptographic use, pseudo-random generator is acceptable here + start_seq = random.choice(self.words) # noqa: S311 late_word = start_seq list_word = [] list_word.append(start_seq) @@ -181,7 +182,8 @@ def gen_sentence( p2 = [j for j in probs if j >= prob] if len(p2) == 0: break - items = temp[probs.index(random.choice(p2))] + # Non-cryptographic use, pseudo-random generator is acceptable here + items = temp[probs.index(random.choice(p2))] # noqa: S311 late_word = items[-1] list_word.append(late_word) @@ -252,7 +254,8 @@ def gen_sentence( # output: 'ยังทำตัวเป็นเซิร์ฟเวอร์คือ' """ if not start_seq: - start_seq = random.choice(self.bi_keys) + # Non-cryptographic use, pseudo-random generator is acceptable here + start_seq = random.choice(self.bi_keys) # noqa: S311 late_word = start_seq list_word = [] list_word.append(start_seq) @@ -270,7 +273,8 @@ def gen_sentence( p2 = [j for j in probs if j >= prob] if len(p2) == 0: break - items = temp[probs.index(random.choice(p2))] + # Non-cryptographic use, pseudo-random generator is acceptable here + items = temp[probs.index(random.choice(p2))] # noqa: S311 late_word = items[1:] list_word.append(late_word) diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index e33d20b18..2e3620454 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -110,7 +110,8 @@ def gen_sentence( # output: 'แมว คุณหลวง ' """ if not start_seq: - start_seq = random.choice(list(thwiki_itos)) + # Non-cryptographic use, pseudo-random generator is acceptable here + start_seq = random.choice(list(thwiki_itos)) # noqa: S311 list_word = learn.predict( start_seq, N, temperature=0.8, min_p=prob, sep="-*-" ).split("-*-") diff --git a/pythainlp/generate/wangchanglm.py b/pythainlp/generate/wangchanglm.py index 12289e338..51ef44261 100644 --- a/pythainlp/generate/wangchanglm.py +++ b/pythainlp/generate/wangchanglm.py @@ -11,7 +11,7 @@ class WangChanGLM: def __init__(self): self.exclude_pattern = re.compile(r"[^ก-๙]+") - self.stop_token = "\n" + self.stop_token = "\n" # noqa: S105 self.PROMPT_DICT = { "prompt_input": ( ": {input}\n: {instruction}\n: " diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index 0619b16b0..c3806e1e2 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -29,7 +29,7 @@ def __init__(self): self._TK_URL, self._TK_END, ) = " ".split() - self.SPACE_SPECIAL_TOKEN = "<_>" + self.SPACE_SPECIAL_TOKEN = "<_>" # noqa: S105 def replace_url(self, text: str) -> str: """Replace url in `text` with TK_URL (https://stackoverflow.com/a/6041965) @@ -60,25 +60,13 @@ def rm_brackets(self, text: str) -> str: new_line = re.sub(r"\{[^a-zA-Z0-9ก-๙]+\}", "", new_line) new_line = re.sub(r"\[[^a-zA-Z0-9ก-๙]+\]", "", new_line) # artifiacts after ( - new_line = re.sub( - r"(?<=\()[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line - ) - new_line = re.sub( - r"(?<=\{)[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line - ) - new_line = re.sub( - r"(?<=\[)[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line - ) + new_line = re.sub(r"(?<=\()[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line) + new_line = re.sub(r"(?<=\{)[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line) + new_line = re.sub(r"(?<=\[)[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line) # artifacts before ) - new_line = re.sub( - r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\))", "", new_line - ) - new_line = re.sub( - r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\})", "", new_line - ) - new_line = re.sub( - r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\])", "", new_line - ) + new_line = re.sub(r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\))", "", new_line) + new_line = re.sub(r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\})", "", new_line) + new_line = re.sub(r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\])", "", new_line) return new_line def replace_newlines(self, text: str) -> str: @@ -103,7 +91,7 @@ def rm_useless_spaces(self, text: str) -> str: """ return re.sub(" {2,}", " ", text) - def replace_spaces(self, text: str, space_token: str = "<_>") -> str: + def replace_spaces(self, text: str, space_token: str = "<_>") -> str: # noqa: S107 """Replace spaces with _ :param str text: text to replace spaces :return: text where all spaces replaced with _ @@ -206,9 +194,7 @@ def __init__(self) -> None: ) self.tokenizer = AutoTokenizer.from_pretrained(_model_name) - self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained( - _model_name - ) + self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained(_model_name) self.model = pipeline( "fill-mask", tokenizer=self.tokenizer, @@ -223,15 +209,17 @@ def generate( max_length: int = 3, sample: bool = False, ) -> str: + """Generate text from PhayaThaiBERT""" sample_txt = sample_text final_text = "" - for j in range(max_length): - input = self.processor.preprocess(sample_txt) + for _ in range(max_length): + input_text = self.processor.preprocess(sample_txt) if sample: - random_word_idx = random.randint(0, 4) - output = self.model(input)[random_word_idx]["sequence"] + # Non-cryptographic use, pseudo-random generator is acceptable here + random_word_idx = random.randint(0, 4) # noqa: S311 + output = self.model(input_text)[random_word_idx]["sequence"] else: - output = self.model(input)[word_rank]["sequence"] + output = self.model(input_text)[word_rank]["sequence"] sample_txt = output + "" final_text = sample_txt @@ -279,9 +267,7 @@ def augment( rank, sample=sample, ) - processed_text = re.sub( - "<_>", " ", self.processor.preprocess(gen_text) - ) + processed_text = re.sub("<_>", " ", self.processor.preprocess(gen_text)) augment_list.append(processed_text) else: raise ValueError( @@ -383,7 +369,8 @@ def get_ner( if pos: warnings.warn( "This model doesn't support output \ - postag and It doesn't output the postag." + postag and It doesn't output the postag.", + stacklevel=2, ) sample_output = [] diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 9909c0c30..9981fda64 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -58,8 +58,8 @@ def middle_cut(sentences: list[str]) -> list[str]: white_space_index = [] white_space_diff = {} - for j, token in enumerate(tokens): - if token == " ": + for j, tok in enumerate(tokens): + if tok == " ": white_space_index.append(j) for white_space in white_space_index: @@ -165,14 +165,14 @@ def split_into_sentences(self, text: str, isMiddleCut: bool = False) -> list[str last_position = len(tokens) pop_split_position = [] split_position = [] - for i, token in enumerate(tokens): - if token == "และ": + for i, tok in enumerate(tokens): + if tok == "และ": and_position = i if ( and_position != -1 and i > and_position - and token == " " + and tok == " " and nearest_space_position == -1 ): if i - and_position != 1: @@ -204,13 +204,13 @@ def split_into_sentences(self, text: str, isMiddleCut: bool = False) -> list[str last_position = len(tokens) pop_split_position = [] split_position = [] - for i, token in enumerate(tokens): - if token == "หรือ": + for i, tok in enumerate(tokens): + if tok == "หรือ": or_position = i if ( or_position != -1 and i > or_position - and token == " " + and tok == " " and nearest_space_position == -1 ): if i - or_position != 1: @@ -242,13 +242,13 @@ def split_into_sentences(self, text: str, isMiddleCut: bool = False) -> list[str pop_split_position = [] last_position = len(tokens) split_position = [] - for i, token in enumerate(tokens): - if token == "จึง": + for i, tok in enumerate(tokens): + if tok == "จึง": cung_position = i if ( cung_position != -1 - and token == " " + and tok == " " and i > cung_position and nearest_space_position == -1 ): diff --git a/pythainlp/tools/misspell.py b/pythainlp/tools/misspell.py index 89c445eae..d2109685b 100644 --- a/pythainlp/tools/misspell.py +++ b/pythainlp/tools/misspell.py @@ -135,7 +135,8 @@ def misspell(sentence: str, ratio: float = 0.05): if potential_candidates is None: continue - candidate = random.choice(potential_candidates) + # Non-cryptographic use, pseudo-random generator is acceptable here + candidate = random.choice(potential_candidates) # noqa: S311 misspelled[pos] = candidate diff --git a/pythainlp/translate/tokenization_small100.py b/pythainlp/translate/tokenization_small100.py index 96d641179..40b284bd6 100644 --- a/pythainlp/translate/tokenization_small100.py +++ b/pythainlp/translate/tokenization_small100.py @@ -127,11 +127,11 @@ def __init__( vocab_file, spm_file, tgt_lang=None, - bos_token="", - eos_token="", - sep_token="", - pad_token="", - unk_token="", + bos_token="", # noqa: S107 + eos_token="", # noqa: S107 + sep_token="", # noqa: S107 + pad_token="", # noqa: S107 + unk_token="", # noqa: S107 language_codes="m2m100", sp_model_kwargs: dict[str, Any] | None = None, num_madeup_words=8, diff --git a/pythainlp/translate/zh_th.py b/pythainlp/translate/zh_th.py index 84a3b75b5..38b5a5f9b 100644 --- a/pythainlp/translate/zh_th.py +++ b/pythainlp/translate/zh_th.py @@ -79,6 +79,8 @@ def __init__( use_gpu: bool = False, pretrained: str = "Lalita/marianmt-zh_cn-th", ) -> None: + from transformers import AutoModelForSeq2SeqLM, AutoTokenizer + self.tokenizer_zhth = AutoTokenizer.from_pretrained(pretrained) self.model_zhth = AutoModelForSeq2SeqLM.from_pretrained(pretrained) if use_gpu: diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index d7a2d6b0e..cae9b0c70 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Romanization of Thai words based on machine-learnt engine ("thai2rom") -""" +"""Romanization of Thai words based on machine-learnt engine ("thai2rom")""" from __future__ import annotations @@ -44,9 +43,7 @@ def __init__(self): # Restore the model and construct the encoder and decoder. self._encoder = Encoder(INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT) - self._decoder = AttentionDecoder( - OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT - ) + self._decoder = AttentionDecoder(OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT) self._network = Seq2Seq( self._encoder, @@ -60,8 +57,7 @@ def __init__(self): self._network.eval() def _prepare_sequence_in(self, text: str): - """Prepare input sequence for PyTorch - """ + """Prepare input sequence for PyTorch""" idxs = [] for ch in text: if ch in self._char_to_ix: @@ -79,9 +75,7 @@ def romanize(self, text: str) -> str: """ input_tensor = self._prepare_sequence_in(text).view(1, -1) input_length = torch.Tensor([len(text) + 1]).int() - target_tensor_logits = self._network( - input_tensor, input_length, None, 0 - ) + target_tensor_logits = self._network(input_tensor, input_length, None, 0) # Seq2seq model returns as the first token, # As a result, target_tensor_logits.size() is torch.Size([0]) @@ -89,10 +83,7 @@ def romanize(self, text: str) -> str: target = [""] else: target_tensor = ( - torch.argmax(target_tensor_logits.squeeze(1), 1) - .cpu() - .detach() - .numpy() + torch.argmax(target_tensor_logits.squeeze(1), 1).cpu().detach().numpy() ) target = [self._ix_to_target_char[t] for t in target_tensor] @@ -100,15 +91,11 @@ def romanize(self, text: str) -> str: class Encoder(nn.Module): - def __init__( - self, vocabulary_size, embedding_size, hidden_size, dropout=0.5 - ): + def __init__(self, vocabulary_size, embedding_size, hidden_size, dropout=0.5): """Constructor""" super().__init__() self.hidden_size = hidden_size - self.character_embedding = nn.Embedding( - vocabulary_size, embedding_size - ) + self.character_embedding = nn.Embedding(vocabulary_size, embedding_size) self.rnn = nn.LSTM( input_size=embedding_size, hidden_size=hidden_size // 2, @@ -180,9 +167,9 @@ def __init__(self, method, hidden_size): def forward(self, hidden, encoder_outputs, mask): # Calculate energies for each encoder output if self.method == "dot": - attn_energies = torch.bmm( - encoder_outputs, hidden.transpose(1, 2) - ).squeeze(2) + attn_energies = torch.bmm(encoder_outputs, hidden.transpose(1, 2)).squeeze( + 2 + ) elif self.method == "general": attn_energies = self.attn( encoder_outputs.view(-1, encoder_outputs.size(-1)) @@ -190,7 +177,9 @@ def forward(self, hidden, encoder_outputs, mask): attn_energies = torch.bmm( attn_energies.view(*encoder_outputs.size()), hidden.transpose(1, 2), - ).squeeze(2) # (batch_size, sequence_len) + ).squeeze( + 2 + ) # (batch_size, sequence_len) elif self.method == "concat": attn_energies = self.attn( torch.cat( @@ -210,16 +199,12 @@ def forward(self, hidden, encoder_outputs, mask): class AttentionDecoder(nn.Module): - def __init__( - self, vocabulary_size, embedding_size, hidden_size, dropout=0.5 - ): + def __init__(self, vocabulary_size, embedding_size, hidden_size, dropout=0.5): """Constructor""" super().__init__() self.vocabulary_size = vocabulary_size self.hidden_size = hidden_size - self.character_embedding = nn.Embedding( - vocabulary_size, embedding_size - ) + self.character_embedding = nn.Embedding(vocabulary_size, embedding_size) self.rnn = nn.LSTM( input_size=embedding_size + self.hidden_size, hidden_size=hidden_size, @@ -296,9 +281,7 @@ def forward( max_len = self.max_length target_vocab_size = self.decoder.vocabulary_size - outputs = torch.zeros(max_len, batch_size, target_vocab_size).to( - device - ) + outputs = torch.zeros(max_len, batch_size, target_vocab_size).to(device) if target_seq is None: assert teacher_forcing_ratio == 0, "Must be zero during inference" @@ -306,14 +289,10 @@ def forward( else: inference = False - encoder_outputs, encoder_hidden = self.encoder( - source_seq, source_seq_len - ) + encoder_outputs, encoder_hidden = self.encoder(source_seq, source_seq_len) decoder_input = ( - torch.tensor([[start_token] * batch_size]) - .view(batch_size, 1) - .to(device) + torch.tensor([[start_token] * batch_size]).view(batch_size, 1).to(device) ) encoder_hidden_h_t = torch.cat( @@ -332,7 +311,8 @@ def forward( _, topi = decoder_output.topk(1) outputs[di] = decoder_output.to(device) - teacher_force = random.random() < teacher_forcing_ratio + # Non-cryptographic use, pseudo-random generator is acceptable here + teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 decoder_input = ( target_seq[:, di].reshape(batch_size, 1) diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index dea1f3306..e3d5c85e5 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -22,8 +22,7 @@ class ThaiG2P: - """Latin transliteration of Thai words, using International Phonetic Alphabet - """ + """Latin transliteration of Thai words, using International Phonetic Alphabet""" def __init__(self): # get the model, download it if it's not available locally @@ -45,9 +44,7 @@ def __init__(self): # Restore the model and construct the encoder and decoder. self._encoder = Encoder(INPUT_DIM, E_EMB_DIM, E_HID_DIM, E_DROPOUT) - self._decoder = AttentionDecoder( - OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT - ) + self._decoder = AttentionDecoder(OUTPUT_DIM, D_EMB_DIM, D_HID_DIM, D_DROPOUT) self._network = Seq2Seq( self._encoder, @@ -61,8 +58,7 @@ def __init__(self): self._network.eval() def _prepare_sequence_in(self, text: str): - """Prepare input sequence for PyTorch. - """ + """Prepare input sequence for PyTorch.""" idxs = [] for ch in text: if ch in self._char_to_ix: @@ -81,9 +77,7 @@ def g2p(self, text: str) -> str: input_tensor = self._prepare_sequence_in(text).view(1, -1) input_length = [len(text) + 1] - target_tensor_logits = self._network( - input_tensor, input_length, None, 0 - ) + target_tensor_logits = self._network(input_tensor, input_length, None, 0) # Seq2seq model returns as the first token, # As a result, target_tensor_logits.size() is torch.Size([0]) @@ -91,10 +85,7 @@ def g2p(self, text: str) -> str: target = [""] else: target_tensor = ( - torch.argmax(target_tensor_logits.squeeze(1), 1) - .cpu() - .detach() - .numpy() + torch.argmax(target_tensor_logits.squeeze(1), 1).cpu().detach().numpy() ) target = [self._ix_to_target_char[t] for t in target_tensor] @@ -102,15 +93,11 @@ def g2p(self, text: str) -> str: class Encoder(nn.Module): - def __init__( - self, vocabulary_size, embedding_size, hidden_size, dropout=0.5 - ): + def __init__(self, vocabulary_size, embedding_size, hidden_size, dropout=0.5): """Constructor""" super().__init__() self.hidden_size = hidden_size - self.character_embedding = nn.Embedding( - vocabulary_size, embedding_size - ) + self.character_embedding = nn.Embedding(vocabulary_size, embedding_size) self.rnn = nn.LSTM( input_size=embedding_size, hidden_size=hidden_size // 2, @@ -184,9 +171,9 @@ def __init__(self, method, hidden_size): def forward(self, hidden, encoder_outputs, mask): # Calculate energies for each encoder output if self.method == "dot": - attn_energies = torch.bmm( - encoder_outputs, hidden.transpose(1, 2) - ).squeeze(2) + attn_energies = torch.bmm(encoder_outputs, hidden.transpose(1, 2)).squeeze( + 2 + ) elif self.method == "general": attn_energies = self.attn( encoder_outputs.view(-1, encoder_outputs.size(-1)) @@ -194,7 +181,9 @@ def forward(self, hidden, encoder_outputs, mask): attn_energies = torch.bmm( attn_energies.view(*encoder_outputs.size()), hidden.transpose(1, 2), - ).squeeze(2) # (batch_size, sequence_len) + ).squeeze( + 2 + ) # (batch_size, sequence_len) elif self.method == "concat": attn_energies = self.attn( torch.cat( @@ -214,16 +203,12 @@ def forward(self, hidden, encoder_outputs, mask): class AttentionDecoder(nn.Module): - def __init__( - self, vocabulary_size, embedding_size, hidden_size, dropout=0.5 - ): + def __init__(self, vocabulary_size, embedding_size, hidden_size, dropout=0.5): """Constructor""" super().__init__() self.vocabulary_size = vocabulary_size self.hidden_size = hidden_size - self.character_embedding = nn.Embedding( - vocabulary_size, embedding_size - ) + self.character_embedding = nn.Embedding(vocabulary_size, embedding_size) self.rnn = nn.LSTM( input_size=embedding_size + self.hidden_size, hidden_size=hidden_size, @@ -300,9 +285,7 @@ def forward( max_len = self.max_length target_vocab_size = self.decoder.vocabulary_size - outputs = torch.zeros(max_len, batch_size, target_vocab_size).to( - device - ) + outputs = torch.zeros(max_len, batch_size, target_vocab_size).to(device) if target_seq is None: assert teacher_forcing_ratio == 0, "Must be zero during inference" @@ -310,14 +293,10 @@ def forward( else: inference = False - encoder_outputs, encoder_hidden = self.encoder( - source_seq, source_seq_len - ) + encoder_outputs, encoder_hidden = self.encoder(source_seq, source_seq_len) decoder_input = ( - torch.tensor([[start_token] * batch_size]) - .view(batch_size, 1) - .to(device) + torch.tensor([[start_token] * batch_size]).view(batch_size, 1).to(device) ) encoder_hidden_h_t = torch.cat( @@ -336,7 +315,8 @@ def forward( _, topi = decoder_output.topk(1) outputs[di] = decoder_output.to(device) - teacher_force = random.random() < teacher_forcing_ratio + # Non-cryptographic use, pseudo-random generator is acceptable here + teacher_force = random.random() < teacher_forcing_ratio # noqa: S311 decoder_input = ( target_seq[:, di].reshape(batch_size, 1) diff --git a/pythainlp/util/__init__.py b/pythainlp/util/__init__.py index 3f1eb8520..cedec8aeb 100644 --- a/pythainlp/util/__init__.py +++ b/pythainlp/util/__init__.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Utility functions, like date conversion and digit conversion -""" +"""Utility functions, like date conversion and digit conversion""" __all__ = [ "Trie", @@ -70,7 +69,7 @@ "analyze_thai_text", ] -from pythainlp.util import spell_words +from pythainlp.util import spell_words # noqa: I001 - keep block order to avoid circular imports from pythainlp.util.abbreviation import abbreviation_to_full_text from pythainlp.util.collate import collate from pythainlp.util.date import ( @@ -133,13 +132,13 @@ # sound_syllable and pronounce have to be imported last, # to prevent circular import issues. # Other imports should be above this line, sorted. -from pythainlp.util.syllable import ( +from pythainlp.util.syllable import ( # noqa: I001 sound_syllable, syllable_length, syllable_open_close_detector, tone_detector, ) -from pythainlp.util.pronounce import ( +from pythainlp.util.pronounce import ( # noqa: I001 rhyme, spelling, thai_consonant_to_spelling, diff --git a/pythainlp/util/strftime.py b/pythainlp/util/strftime.py index 21a18af24..fa6d6dae4 100644 --- a/pythainlp/util/strftime.py +++ b/pythainlp/util/strftime.py @@ -1,8 +1,7 @@ # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project # SPDX-FileType: SOURCE # SPDX-License-Identifier: Apache-2.0 -"""Thai date/time formatting. -""" +"""Thai date/time formatting.""" from __future__ import annotations @@ -30,8 +29,7 @@ def _std_strftime(dt_obj: datetime, fmt_char: str) -> str: - """Standard datetime.strftime() with normalization and exception handling. - """ + """Standard datetime.strftime() with normalization and exception handling.""" str_ = "" try: str_ = dt_obj.strftime(f"%{fmt_char}") @@ -49,7 +47,8 @@ def _std_strftime(dt_obj: datetime, fmt_char: str) -> str: f"The system raises this ValueError: {err}\n" f"Continue working without the directive." ), - UserWarning, + category=UserWarning, + stacklevel=2, ) str_ = fmt_char return str_ @@ -107,9 +106,7 @@ def _thai_strftime(dt_obj: datetime, fmt_char: str) -> str: elif fmt_char == "g": # Same year as in ``%G'', # but as a decimal number without century (00-99). - str_ = ( - str(int(dt_obj.strftime("%G")) + _BE_AD_DIFFERENCE)[-2:] - ).zfill(2) + str_ = (str(int(dt_obj.strftime("%G")) + _BE_AD_DIFFERENCE)[-2:]).zfill(2) elif fmt_char == "v": # BSD extension, ' 6-Oct-1976' str_ = f"{dt_obj.day:>2}-{thai_abbr_months[dt_obj.month - 1]}-{str(dt_obj.year + _BE_AD_DIFFERENCE).zfill(4)}" diff --git a/pythainlp/wangchanberta/core.py b/pythainlp/wangchanberta/core.py index d07537f32..9d2583b33 100644 --- a/pythainlp/wangchanberta/core.py +++ b/pythainlp/wangchanberta/core.py @@ -22,9 +22,7 @@ class ThaiNameTagger: - def __init__( - self, dataset_name: str = "thainer", grouped_entities: bool = True - ): + def __init__(self, dataset_name: str = "thainer", grouped_entities: bool = True): """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ @@ -71,7 +69,8 @@ def get_ner( """ if pos: warnings.warn( - "This model doesn't support output of POS tags and it doesn't output the POS tags." + "This model doesn't support output of POS tags and it doesn't output the POS tags.", + stacklevel=2, ) text = re.sub(" ", "<_>", text) self.json_ner = self.classify_tokens(text) @@ -102,9 +101,7 @@ def get_ner( self.sent_ner = self.sent_ner[1:] for idx, (word, ner) in enumerate(self.sent_ner): if idx > 0 and ner.startswith("B-"): - if self._clear_tag(ner) == self._clear_tag( - self.sent_ner[idx - 1][1] - ): + if self._clear_tag(ner) == self._clear_tag(self.sent_ner[idx - 1][1]): self.sent_ner[idx] = (word, ner.replace("B-", "I-")) if tag: temp = "" @@ -131,9 +128,7 @@ def get_ner( class NamedEntityRecognition: - def __init__( - self, model: str = "pythainlp/thainer-corpus-v2-base-model" - ) -> None: + def __init__(self, model: str = "pythainlp/thainer-corpus-v2-base-model") -> None: """This function tags named entities in text in IOB format. Powered by wangchanberta from VISTEC-depa\ @@ -180,7 +175,8 @@ def get_ner( if pos: warnings.warn( - "This model doesn't support output postag and It doesn't output the postag." + "This model doesn't support output postag and It doesn't output the postag.", + stacklevel=2, ) words_token = word_tokenize(text.replace(" ", "<_>")) inputs = self.tokenizer( @@ -195,9 +191,7 @@ def get_ner( predicted_token_class = [ self.model.config.id2label[t.item()] for t in predictions[0] ] - ner_tag = self._fix_span_error( - inputs["input_ids"][0], predicted_token_class - ) + ner_tag = self._fix_span_error(inputs["input_ids"][0], predicted_token_class) if tag: temp = "" sent = ""