Merge pull request #1195 from bact/add-more-lint-rules

Add more lint rules and fix some

Author: bact

.github/workflows/lint.yml

@@ -26,4 +26,4 @@ jobs:
       uses: astral-sh/ruff-action@v3
       with:
         src: "./pythainlp ./tests ./examples"
-        args: check --fix --verbose --line-length 79 --select I,W,C901,W291,W293
+        args: check --verbose --config pyproject.toml

pyproject.toml

@@ -287,18 +287,36 @@ source = ["pythainlp"]
 
 # Ruff configuration
 [tool.ruff]
-line-length = 79
 indent-width = 4
+line-length = 79
 target-version = "py39"
 
 [tool.ruff.format]
-quote-style = "double"
+docstring-code-format = true
 indent-style = "space"
-skip-magic-trailing-comma = false
 line-ending = "auto"
-docstring-code-format = true
+quote-style = "double"
+skip-magic-trailing-comma = false
 
-[tool.ruff.lint.mccabe]
+[tool.ruff.lint]
 # Flag errors (`C901`) whenever the complexity level exceeds 5. Default is 10.
 # We should aim to gradually reduce this to 10.
-max-complexity = 38
+mccabe.max-complexity = 38
+select = [
+    "E",
+    "F",
+    "I",
+    "W",
+    "S",
+    "B028",
+    "C901",
+    "F401",
+    "F811",
+    "F841",
+    "W291",
+    "W293",
+]
+# Some rules are ignored for now; we can consider enabling them in the future.
+# F403 and F405 are ignored due to thai2fit/ulmfit module's star imports.
+# S101 is use of assert statement, should be an easy fix.
+extend-ignore = ["E402", "E501", "E722", "F403", "F405", "S101", "S202", "S301", "S310"]

pythainlp/augment/lm/phayathaibert.py

@@ -20,9 +20,7 @@ def __init__(self) -> None:
         )
 
         self.tokenizer = AutoTokenizer.from_pretrained(_MODEL_NAME)
-        self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained(
-            _MODEL_NAME
-        )
+        self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained(_MODEL_NAME)
         self.model = pipeline(
             "fill-mask",
             tokenizer=self.tokenizer,
@@ -40,23 +38,22 @@ def generate(
         sample_txt = sample_text
         final_text = ""
 
-        for j in range(max_length):
-            input = self.processor.preprocess(sample_txt)
+        for _ in range(max_length):
+            input_text = self.processor.preprocess(sample_txt)
             if sample:
-                random_word_idx = random.randint(0, 4)
-                output = self.model(input)[random_word_idx]["sequence"]
+                # Non-cryptographic use, pseudo-random generator is acceptable here
+                random_word_idx = random.randint(0, 4)  # noqa: S311
+                output = self.model(input_text)[random_word_idx]["sequence"]
             else:
-                output = self.model(input)[word_rank]["sequence"]
+                output = self.model(input_text)[word_rank]["sequence"]
             sample_txt = output + "<mask>"
             final_text = sample_txt
 
         gen_txt = re.sub("<mask>", "", final_text)
 
         return gen_txt
 
-    def augment(
-        self, text: str, num_augs: int = 3, sample: bool = False
-    ) -> list[str]:
+    def augment(self, text: str, num_augs: int = 3, sample: bool = False) -> list[str]:
         """Text augmentation from PhayaThaiBERT
 
         :param str text: Thai text
@@ -90,9 +87,7 @@ def augment(
         if num_augs <= MAX_NUM_AUGS:
             for rank in range(num_augs):
                 gen_text = self.generate(text, rank, sample=sample)
-                processed_text = re.sub(
-                    "<_>", " ", self.processor.preprocess(gen_text)
-                )
+                processed_text = re.sub("<_>", " ", self.processor.preprocess(gen_text))
                 augment_list.append(processed_text)
         else:
             raise ValueError(

pythainlp/cli/tokenize.py

@@ -1,8 +1,7 @@
 # SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
 # SPDX-FileType: SOURCE
 # SPDX-License-Identifier: Apache-2.0
-"""Command line for PyThaiNLP's tokenizers.
-"""
+"""Command line for PyThaiNLP's tokenizers."""
 
 import argparse
 
@@ -17,10 +16,10 @@
 )
 from pythainlp.tools import safe_print
 
-DEFAULT_SENT_TOKEN_SEPARATOR = "@@"
-DEFAULT_SUBWORD_TOKEN_SEPARATOR = "/"
-DEFAULT_SYLLABLE_TOKEN_SEPARATOR = "~"
-DEFAULT_WORD_TOKEN_SEPARATOR = "|"
+DEFAULT_SENT_TOKEN_SEPARATOR = "@@"  # noqa: S105
+DEFAULT_SUBWORD_TOKEN_SEPARATOR = "/"  # noqa: S105
+DEFAULT_SYLLABLE_TOKEN_SEPARATOR = "~"  # noqa: S105
+DEFAULT_WORD_TOKEN_SEPARATOR = "|"  # noqa: S105
 
 
 class SubAppBase:

pythainlp/corpus/__init__.py

@@ -71,25 +71,23 @@
 
 
 def corpus_path() -> str:
-    """Get path where corpus files are kept locally.
-    """
+    """Get path where corpus files are kept locally."""
     return _CORPUS_PATH
 
 
 def corpus_db_url() -> str:
-    """Get remote URL of corpus catalog.
-    """
+    """Get remote URL of corpus catalog."""
     return _CORPUS_DB_URL
 
 
 def corpus_db_path() -> str:
-    """Get local path of corpus catalog.
-    """
+    """Get local path of corpus catalog."""
     return _CORPUS_DB_PATH
 
-# DO NOT REORDER these pythainlp.corpus imports.
+
+# DO NOT REORDER these pythainlp.corpus.core imports.
 # These imports must come before other pythainlp.corpus.* imports
-from pythainlp.corpus.core import (
+from pythainlp.corpus.core import (  # noqa: I001
     download,
     get_corpus,
     get_corpus_as_is,

pythainlp/corpus/core.py

@@ -322,7 +322,8 @@ def _check_hash(dst: str, md5: str) -> None:
 
         with open(get_full_data_path(dst), "rb") as f:
             content = f.read()
-            file_md5 = hashlib.md5(content).hexdigest()
+            # MD5 is insecure but sufficient here
+            file_md5 = hashlib.md5(content).hexdigest()  # noqa: S324
 
             if md5 != file_md5:
                 raise ValueError("Hash does not match expected.")

pythainlp/generate/core.py

@@ -70,16 +70,15 @@ def gen_sentence(
             # output: 'แมวเวลานะนั้น'
         """
         if not start_seq:
-            start_seq = random.choice(self.word)
+            # Non-cryptographic use, pseudo-random generator is acceptable here
+            start_seq = random.choice(self.word)  # noqa: S311
         rand_text = start_seq.lower()
         self._word_prob = {
             i: self.counts[i] / self.n
             for i in self.word
             if self.counts[i] / self.n >= prob
         }
-        return self._next_word(
-            rand_text, N, output_str, prob=prob, duplicate=duplicate
-        )
+        return self._next_word(rand_text, N, output_str, prob=prob, duplicate=duplicate)
 
     def _next_word(
         self,
@@ -95,10 +94,11 @@ def _next_word(
         if N > len(word_list):
             N = len(word_list)
         for _ in range(N):
-            w = random.choice(word_list)
+            # Non-cryptographic use, pseudo-random generator is acceptable here
+            w = random.choice(word_list)  # noqa: S311
             if duplicate is False:
                 while w in words:
-                    w = random.choice(word_list)
+                    w = random.choice(word_list)  # noqa: S311
             words.append(w)
 
         if output_str:
@@ -163,7 +163,8 @@ def gen_sentence(
             # output: 'แมวไม่ได้รับเชื้อมัน'
         """
         if not start_seq:
-            start_seq = random.choice(self.words)
+            # Non-cryptographic use, pseudo-random generator is acceptable here
+            start_seq = random.choice(self.words)  # noqa: S311
         late_word = start_seq
         list_word = []
         list_word.append(start_seq)
@@ -181,7 +182,8 @@ def gen_sentence(
             p2 = [j for j in probs if j >= prob]
             if len(p2) == 0:
                 break
-            items = temp[probs.index(random.choice(p2))]
+            # Non-cryptographic use, pseudo-random generator is acceptable here
+            items = temp[probs.index(random.choice(p2))]  # noqa: S311
             late_word = items[-1]
             list_word.append(late_word)
 
@@ -252,7 +254,8 @@ def gen_sentence(
             # output: 'ยังทำตัวเป็นเซิร์ฟเวอร์คือ'
         """
         if not start_seq:
-            start_seq = random.choice(self.bi_keys)
+            # Non-cryptographic use, pseudo-random generator is acceptable here
+            start_seq = random.choice(self.bi_keys)  # noqa: S311
         late_word = start_seq
         list_word = []
         list_word.append(start_seq)
@@ -270,7 +273,8 @@ def gen_sentence(
             p2 = [j for j in probs if j >= prob]
             if len(p2) == 0:
                 break
-            items = temp[probs.index(random.choice(p2))]
+            # Non-cryptographic use, pseudo-random generator is acceptable here
+            items = temp[probs.index(random.choice(p2))]  # noqa: S311
             late_word = items[1:]
             list_word.append(late_word)
 

pythainlp/generate/thai2fit.py

@@ -110,7 +110,8 @@ def gen_sentence(
       # output: 'แมว คุณหลวง '
     """
     if not start_seq:
-        start_seq = random.choice(list(thwiki_itos))
+        # Non-cryptographic use, pseudo-random generator is acceptable here
+        start_seq = random.choice(list(thwiki_itos))  # noqa: S311
     list_word = learn.predict(
         start_seq, N, temperature=0.8, min_p=prob, sep="-*-"
     ).split("-*-")

pythainlp/generate/wangchanglm.py

@@ -11,7 +11,7 @@
 class WangChanGLM:
     def __init__(self):
         self.exclude_pattern = re.compile(r"[^ก-๙]+")
-        self.stop_token = "\n"
+        self.stop_token = "\n"  # noqa: S105
         self.PROMPT_DICT = {
             "prompt_input": (
                 "<context>: {input}\n<human>: {instruction}\n<bot>: "

pythainlp/phayathaibert/core.py

@@ -29,7 +29,7 @@ def __init__(self):
             self._TK_URL,
             self._TK_END,
         ) = "<unk> <rep> <wrep> <url> </s>".split()
-        self.SPACE_SPECIAL_TOKEN = "<_>"
+        self.SPACE_SPECIAL_TOKEN = "<_>"  # noqa: S105
 
     def replace_url(self, text: str) -> str:
         """Replace url in `text` with TK_URL (https://stackoverflow.com/a/6041965)
@@ -60,25 +60,13 @@ def rm_brackets(self, text: str) -> str:
         new_line = re.sub(r"\{[^a-zA-Z0-9ก-๙]+\}", "", new_line)
         new_line = re.sub(r"\[[^a-zA-Z0-9ก-๙]+\]", "", new_line)
         # artifiacts after (
-        new_line = re.sub(
-            r"(?<=\()[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line
-        )
-        new_line = re.sub(
-            r"(?<=\{)[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line
-        )
-        new_line = re.sub(
-            r"(?<=\[)[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line
-        )
+        new_line = re.sub(r"(?<=\()[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line)
+        new_line = re.sub(r"(?<=\{)[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line)
+        new_line = re.sub(r"(?<=\[)[^a-zA-Z0-9ก-๙]+(?=[a-zA-Z0-9ก-๙])", "", new_line)
         # artifacts before )
-        new_line = re.sub(
-            r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\))", "", new_line
-        )
-        new_line = re.sub(
-            r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\})", "", new_line
-        )
-        new_line = re.sub(
-            r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\])", "", new_line
-        )
+        new_line = re.sub(r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\))", "", new_line)
+        new_line = re.sub(r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\})", "", new_line)
+        new_line = re.sub(r"(?<=[a-zA-Z0-9ก-๙])[^a-zA-Z0-9ก-๙]+(?=\])", "", new_line)
         return new_line
 
     def replace_newlines(self, text: str) -> str:
@@ -103,7 +91,7 @@ def rm_useless_spaces(self, text: str) -> str:
         """
         return re.sub(" {2,}", " ", text)
 
-    def replace_spaces(self, text: str, space_token: str = "<_>") -> str:
+    def replace_spaces(self, text: str, space_token: str = "<_>") -> str:  # noqa: S107
         """Replace spaces with _
         :param str text: text to replace spaces
         :return: text where all spaces replaced with _
@@ -206,9 +194,7 @@ def __init__(self) -> None:
         )
 
         self.tokenizer = AutoTokenizer.from_pretrained(_model_name)
-        self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained(
-            _model_name
-        )
+        self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained(_model_name)
         self.model = pipeline(
             "fill-mask",
             tokenizer=self.tokenizer,
@@ -223,15 +209,17 @@ def generate(
         max_length: int = 3,
         sample: bool = False,
     ) -> str:
+        """Generate text from PhayaThaiBERT"""
         sample_txt = sample_text
         final_text = ""
-        for j in range(max_length):
-            input = self.processor.preprocess(sample_txt)
+        for _ in range(max_length):
+            input_text = self.processor.preprocess(sample_txt)
             if sample:
-                random_word_idx = random.randint(0, 4)
-                output = self.model(input)[random_word_idx]["sequence"]
+                # Non-cryptographic use, pseudo-random generator is acceptable here
+                random_word_idx = random.randint(0, 4)  # noqa: S311
+                output = self.model(input_text)[random_word_idx]["sequence"]
             else:
-                output = self.model(input)[word_rank]["sequence"]
+                output = self.model(input_text)[word_rank]["sequence"]
             sample_txt = output + "<mask>"
             final_text = sample_txt
 
@@ -279,9 +267,7 @@ def augment(
                     rank,
                     sample=sample,
                 )
-                processed_text = re.sub(
-                    "<_>", " ", self.processor.preprocess(gen_text)
-                )
+                processed_text = re.sub("<_>", " ", self.processor.preprocess(gen_text))
                 augment_list.append(processed_text)
         else:
             raise ValueError(
@@ -383,7 +369,8 @@ def get_ner(
         if pos:
             warnings.warn(
                 "This model doesn't support output \
-                          postag and It doesn't output the postag."
+                          postag and It doesn't output the postag.",
+                stacklevel=2,
             )
 
         sample_output = []