From 9eb561d1506fc4084807a0c362ee559bc1f2c731 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 8 Jan 2026 09:52:55 +0000 Subject: [PATCH 01/15] Initial plan From 5898f2b27df6b0f55505ab369dbba7d2abb8ae01 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:00:14 +0000 Subject: [PATCH 02/15] Optimize Python code for better performance - replace range(len()), list concatenation, and unnecessary list() calls Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com> --- pythainlp/corpus/core.py | 6 +++--- pythainlp/lm/text_util.py | 4 ++-- pythainlp/spell/pn.py | 10 ++++++++-- pythainlp/tokenize/crfcut.py | 23 ++++++++--------------- pythainlp/tokenize/thaisumcut.py | 18 +++++++++--------- pythainlp/util/wordtonum.py | 8 ++++---- 6 files changed, 34 insertions(+), 35 deletions(-) diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index d3372bfe2..dc372f88d 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -195,8 +195,8 @@ def get_corpus_default_db(name: str, version: str = "") -> Union[str, None]: with open(default_db_path, encoding="utf-8-sig") as fh: corpus_db = json.load(fh) - if name in list(corpus_db.keys()): - if version in list(corpus_db[name]["versions"].keys()): + if name in corpus_db: + if version in corpus_db[name]["versions"]: return path_pythainlp_corpus( corpus_db[name]["versions"][version]["filename"] ) @@ -257,7 +257,7 @@ def get_corpus_path( CUSTOMIZE: Dict[str, str] = { # "the corpus name":"path" } - if name in list(CUSTOMIZE): + if name in CUSTOMIZE: return CUSTOMIZE[name] default_path = get_corpus_default_db(name=name, version=version) diff --git a/pythainlp/lm/text_util.py b/pythainlp/lm/text_util.py index 946fd8451..fbe5faaf6 100644 --- a/pythainlp/lm/text_util.py +++ b/pythainlp/lm/text_util.py @@ -25,7 +25,7 @@ def calculate_ngram_counts( ngram_counts = {} for n in range(n_min, n_max + 1): - for i in range(len(list_words) - n + 1): + for i, _ in enumerate(list_words[:-n+1], 0): ngram = tuple(list_words[i:i + n]) ngram_counts[ngram] = ngram_counts.get(ngram, 0) + 1 @@ -56,7 +56,7 @@ def remove_repeated_ngrams(string_list: List[str], n: int = 2) -> List[str]: output_list = [] - for i in range(len(string_list)): + for i, _ in enumerate(string_list): if i + n <= len(string_list): ngram = tuple(string_list[i:i + n]) diff --git a/pythainlp/spell/pn.py b/pythainlp/spell/pn.py index 4cdfab5d7..ff2d7ccae 100644 --- a/pythainlp/spell/pn.py +++ b/pythainlp/spell/pn.py @@ -68,8 +68,14 @@ def _edits1(word: str) -> Set[str]: transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R) > 1] replaces = [L + c + R[1:] for L, R in splits if R for c in thai_letters] inserts = [L + c + R for L, R in splits for c in thai_letters] - - return set(deletes + transposes + replaces + inserts) + + # Use set union for better performance than list concatenation + result = set(deletes) + result.update(transposes) + result.update(replaces) + result.update(inserts) + + return result def _edits2(word: str) -> Set[str]: diff --git a/pythainlp/tokenize/crfcut.py b/pythainlp/tokenize/crfcut.py index 9746d0113..8ffa9e2c5 100644 --- a/pythainlp/tokenize/crfcut.py +++ b/pythainlp/tokenize/crfcut.py @@ -141,25 +141,18 @@ def extract_features( :return: list of lists of features to be fed to CRF """ doc_features = [] - doc = ( - ["xxpad" for i in range(window)] - + doc - + ["xxpad" for i in range(window)] - ) + # Pad the document with "xxpad" tokens efficiently + padded_doc = ["xxpad"] * window + padded_doc.extend(doc) + padded_doc.extend(["xxpad"] * window) + doc = padded_doc # add enders and starters doc_ender = [] doc_starter = [] - for i in range(len(doc)): - if doc[i] in _ENDERS: - doc_ender.append("ender") - else: - doc_ender.append("normal") - - if doc[i] in _STARTERS: - doc_starter.append("starter") - else: - doc_starter.append("normal") + for token in doc: + doc_ender.append("ender" if token in _ENDERS else "normal") + doc_starter.append("starter" if token in _STARTERS else "normal") # for each word for i in range(window, len(doc) - window): diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 2c74f8875..9c92032dd 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -171,14 +171,14 @@ def split_into_sentences( last_position = len(tokens) pop_split_position = [] split_position = [] - for i in range(len(tokens)): - if tokens[i] == "และ": + for i, token in enumerate(tokens): + if token == "และ": and_position = i if ( and_position != -1 and i > and_position - and tokens[i] == " " + and token == " " and nearest_space_position == -1 ): if i - and_position != 1: @@ -210,13 +210,13 @@ def split_into_sentences( last_position = len(tokens) pop_split_position = [] split_position = [] - for i in range(len(tokens)): - if tokens[i] == "หรือ": + for i, token in enumerate(tokens): + if token == "หรือ": or_position = i if ( or_position != -1 and i > or_position - and tokens[i] == " " + and token == " " and nearest_space_position == -1 ): if i - or_position != 1: @@ -248,13 +248,13 @@ def split_into_sentences( pop_split_position = [] last_position = len(tokens) split_position = [] - for i in range(len(tokens)): - if tokens[i] == "จึง": + for i, token in enumerate(tokens): + if token == "จึง": cung_position = i if ( cung_position != -1 - and tokens[i] == " " + and token == " " and i > cung_position and nearest_space_position == -1 ): diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index 2e08d3b6a..51bc5b1af 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -51,7 +51,7 @@ def _check_is_thainum(word: str): - for j in list(_digits.keys()): + for j in _digits: if j in word: return (True, "num") for j in ["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด", "ลบ"]: @@ -60,9 +60,9 @@ def _check_is_thainum(word: str): return (False, None) -_dict_words = [i for i in list(thai_words()) if not _check_is_thainum(i)[0]] -_dict_words += list(_digits.keys()) -_dict_words += ["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"] +_dict_words = [i for i in thai_words() if not _check_is_thainum(i)[0]] +_dict_words.extend(_digits.keys()) +_dict_words.extend(["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"]) _tokenizer_thaiwords = Tokenizer(_dict_words) From 21a86693f3dd8af83c211f4af4169e723bdc29d6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:04:56 +0000 Subject: [PATCH 03/15] Further optimize performance - improve middle_cut string building and khavee loop patterns Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com> --- pythainlp/khavee/core.py | 12 +++++----- pythainlp/tokenize/thaisumcut.py | 38 ++++++++++++++------------------ 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/pythainlp/khavee/core.py b/pythainlp/khavee/core.py index 81e05b662..25dc3b184 100644 --- a/pythainlp/khavee/core.py +++ b/pythainlp/khavee/core.py @@ -460,11 +460,11 @@ def check_klon(self, text: str, k_type: int = 8) -> Union[List[str], str]: ): return "The poem does not have 4 complete sentences." else: - for i in range(len(list_sumpus_sent1)): + for i, sumpus_sent1 in enumerate(list_sumpus_sent1): countwrong = 0 for j in list_sumpus_sent2h[i]: if ( - self.is_sumpus(list_sumpus_sent1[i], j) + self.is_sumpus(sumpus_sent1, j) is False ): countwrong += 1 @@ -473,7 +473,7 @@ def check_klon(self, text: str, k_type: int = 8) -> Union[List[str], str]: "Can't find rhyme between paragraphs " + str( ( - list_sumpus_sent1[i], + sumpus_sent1, list_sumpus_sent2h[i], ) ) @@ -559,11 +559,11 @@ def check_klon(self, text: str, k_type: int = 8) -> Union[List[str], str]: ): return "The poem does not have 4 complete sentences." else: - for i in range(len(list_sumpus_sent1)): + for i, sumpus_sent1 in enumerate(list_sumpus_sent1): countwrong = 0 for j in list_sumpus_sent2h[i]: if ( - self.is_sumpus(list_sumpus_sent1[i], j) + self.is_sumpus(sumpus_sent1, j) is False ): countwrong += 1 @@ -572,7 +572,7 @@ def check_klon(self, text: str, k_type: int = 8) -> Union[List[str], str]: "Can't find rhyme between paragraphs " + str( ( - list_sumpus_sent1[i], + sumpus_sent1, list_sumpus_sent2h[i], ) ) diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 9c92032dd..d909c7d63 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -30,18 +30,21 @@ def list_to_string(list: List[str]) -> str: def middle_cut(sentences: List[str]) -> List[str]: - new_text = "" + result_parts = [] for sentence in sentences: sentence_size = len(word_tokenize(sentence, keep_whitespace=False)) - - for k in range(0, len(sentence)): - if k == 0 or k + 1 >= len(sentence): + + sentence_len = len(sentence) + for k in range(0, sentence_len): + if k == 0 or k + 1 >= sentence_len: continue if sentence[k].isdigit() and sentence[k - 1] == " ": sentence = sentence[: k - 1] + sentence[k:] - if k + 2 <= len(sentence): + sentence_len = len(sentence) # Update length after modification + if k + 2 <= sentence_len: if sentence[k].isdigit() and sentence[k + 1] == " ": sentence = sentence[: k + 1] + sentence[k + 2 :] + sentence_len = len(sentence) # Update length after modification fixed_text_lenth = 20 @@ -53,33 +56,26 @@ def middle_cut(sentences: List[str]) -> List[str]: white_space_index = [] white_space_diff = {} - for j in range(len(tokens)): - if tokens[j] == " ": + for j, token in enumerate(tokens): + if token == " ": white_space_index.append(j) for white_space in white_space_index: - white_space_diff.update( - {white_space: abs(white_space - middle_space)} - ) + white_space_diff[white_space] = abs(white_space - middle_space) - if len(white_space_diff) > 0: + if white_space_diff: min_diff = min( white_space_diff.items(), key=operator.itemgetter(1) ) tokens.pop(min_diff[0]) tokens.insert(min_diff[0], "") - new_text = new_text + list_to_string(tokens) + "" + result_parts.append(list_to_string(tokens)) else: - new_text = new_text + sentence + "" - - sentences = new_text.split("") - sentences = [s.strip() for s in sentences] - if "" in sentences: - sentences.remove("") - if "nan" in sentences: - sentences.remove("nan") + result_parts.append(sentence) - sentences = list(filter(None, sentences)) + sentences = "".join(result_parts).split("") + sentences = [s.strip() for s in sentences if s.strip()] + return sentences From 8c5e818458d956c6b8b0c58b21ae362a6b5f5a0c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:08:53 +0000 Subject: [PATCH 04/15] Address code review feedback - revert unnecessary enumerate usage and fix middle_cut Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com> --- pythainlp/lm/text_util.py | 4 ++-- pythainlp/tokenize/thaisumcut.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pythainlp/lm/text_util.py b/pythainlp/lm/text_util.py index fbe5faaf6..946fd8451 100644 --- a/pythainlp/lm/text_util.py +++ b/pythainlp/lm/text_util.py @@ -25,7 +25,7 @@ def calculate_ngram_counts( ngram_counts = {} for n in range(n_min, n_max + 1): - for i, _ in enumerate(list_words[:-n+1], 0): + for i in range(len(list_words) - n + 1): ngram = tuple(list_words[i:i + n]) ngram_counts[ngram] = ngram_counts.get(ngram, 0) + 1 @@ -56,7 +56,7 @@ def remove_repeated_ngrams(string_list: List[str], n: int = 2) -> List[str]: output_list = [] - for i, _ in enumerate(string_list): + for i in range(len(string_list)): if i + n <= len(string_list): ngram = tuple(string_list[i:i + n]) diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index d909c7d63..3c2624b1d 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -73,8 +73,12 @@ def middle_cut(sentences: List[str]) -> List[str]: else: result_parts.append(sentence) - sentences = "".join(result_parts).split("") - sentences = [s.strip() for s in sentences if s.strip()] + # Split all result parts by and filter + all_sentences = [] + for part in result_parts: + all_sentences.extend(part.split("")) + + sentences = [s.strip() for s in all_sentences if s.strip()] return sentences From 2f805d7799356e34fedad28bb836085934036ac6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 01:33:02 +0000 Subject: [PATCH 05/15] Optimize performance: use dict comprehension, deque for O(1) operations, re.search over re.findall, and efficient list slicing Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/benchmarks/word_tokenization.py | 12 +++++------- pythainlp/corpus/oscar.py | 6 ++---- pythainlp/tokenize/core.py | 15 ++++++++------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index ce9991cfc..851c0ea14 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -58,13 +58,11 @@ def _flatten_result(my_dict: dict, sep: str = ":") -> dict: :return: a one-dimension dictionary with keys combined :rtype: dict[str, float | str] """ - items = [] - for k1, kv2 in my_dict.items(): - for k2, v in kv2.items(): - new_key = f"{k1}{sep}{k2}" - items.append((new_key, v)) - - return dict(items) + return { + f"{k1}{sep}{k2}": v + for k1, kv2 in my_dict.items() + for k2, v in kv2.items() + } def benchmark(ref_samples: list[str], samples: list[str]) -> pd.DataFrame: diff --git a/pythainlp/corpus/oscar.py b/pythainlp/corpus/oscar.py index 0a0b540fc..b13718471 100644 --- a/pythainlp/corpus/oscar.py +++ b/pythainlp/corpus/oscar.py @@ -30,8 +30,7 @@ def word_freqs() -> list[tuple[str, int]]: path = str(path) with open(path, encoding="utf-8-sig") as f: - lines = list(f.readlines()) - del lines[0] + lines = f.readlines()[1:] # Skip first line efficiently for line in lines: temp = line.strip().split(",") if len(temp) >= 2: @@ -54,8 +53,7 @@ def unigram_word_freqs() -> dict[str, int]: path = str(path) with open(path, encoding="utf-8-sig") as fh: - lines = list(fh.readlines()) - del lines[0] + lines = fh.readlines()[1:] # Skip first line efficiently for i in lines: temp = i.strip().split(",") if temp[0] != " " and '"' not in temp[0]: diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index 234be457e..c363619a4 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -9,6 +9,7 @@ import copy import re +from collections import deque from collections.abc import Iterable from pythainlp.tokenize import ( @@ -340,13 +341,13 @@ def indices_words(words): def map_indices_to_words(index_list, sentences): result = [] - c = copy.copy(index_list) + c = deque(index_list) # Use deque for O(1) popleft n_sum = 0 for sentence in sentences: words = sentence sentence_result = [] n = 0 - for start, end in c: + for start, end in list(c): # Create a snapshot to iterate if start > n_sum + len(words) - 1: break else: @@ -357,7 +358,7 @@ def map_indices_to_words(index_list, sentences): result.append(sentence_result) n_sum += len(words) for _ in range(n): - del c[0] + c.popleft() # O(1) operation instead of del c[0] return result @@ -464,7 +465,8 @@ def sent_tokenize( result = [] _temp: list[str] = [] for i, w in enumerate(text): - if re.findall(r" ", w) != [] and re.findall(r"\w", w) == []: + # Use re.search instead of re.findall for boolean checks (faster) + if re.search(r" ", w) and not re.search(r"\w", w): if not _temp: continue result.append(_temp) @@ -480,9 +482,8 @@ def sent_tokenize( result = [] _temp = [] for i, w in enumerate(text): - if ( - re.findall(r"\s", w) != [] or re.findall(r"\n", w) != [] - ) and re.findall(r"\w", w) == []: + # Use re.search instead of re.findall for boolean checks (faster) + if (re.search(r"\s", w) or re.search(r"\n", w)) and not re.search(r"\w", w): if not _temp: continue result.append(_temp) From c0365690b120cfb348ef965686c489c93ebcb0cf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 01:36:00 +0000 Subject: [PATCH 06/15] Remove unnecessary list() calls on dict.keys() and use itertools.chain for efficient iteration Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/ancient/aksonhan.py | 4 +++- pythainlp/wsd/core.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pythainlp/ancient/aksonhan.py b/pythainlp/ancient/aksonhan.py index 15e41e5f3..6786c3ae3 100644 --- a/pythainlp/ancient/aksonhan.py +++ b/pythainlp/ancient/aksonhan.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from itertools import chain + from pythainlp import thai_consonants, thai_tonemarks from pythainlp.corpus import thai_orst_words from pythainlp.tokenize import Tokenizer @@ -17,7 +19,7 @@ _dict_aksonhan[i + i + j + i] = i + "ั" + j + i _dict_aksonhan[i + i] = "ั" + i _set_aksonhan = set(_dict_aksonhan.keys()) -_trie = Trie(list(_dict_aksonhan.keys()) + list(thai_consonants)) +_trie = Trie(chain(_dict_aksonhan.keys(), thai_consonants)) # Use chain instead of list concatenation _tokenizer = Tokenizer(custom_dict=_trie, engine="mm") _dict_thai = set(thai_orst_words()) # call Thai words diff --git a/pythainlp/wsd/core.py b/pythainlp/wsd/core.py index 8b947bf02..98f0be8c8 100644 --- a/pythainlp/wsd/core.py +++ b/pythainlp/wsd/core.py @@ -13,8 +13,8 @@ for i, j in zip(_wsd_dict["word"], _wsd_dict["meaning"]): _mean_all[i] = j -_all_word = set(list(_mean_all.keys())) -_TRIE = Trie(list(_all_word)) +_all_word = set(_mean_all.keys()) # dict.keys() is already iterable, no need for list() +_TRIE = Trie(_all_word) _word_cut = Tokenizer(custom_dict=_TRIE) _MODEL = None From 0e7fc3757a67ffdd4f71a4c1053584f4f2d51efe Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 01:37:44 +0000 Subject: [PATCH 07/15] Address code review: optimize deque usage, compile regex patterns, simplify conditions Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/tokenize/core.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index c363619a4..4a3e94383 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -27,6 +27,11 @@ ) from pythainlp.util.trie import Trie, dict_trie +# Compiled regex patterns for better performance +_RE_WHITESPACE = re.compile(r"\s") +_RE_WORD_CHAR = re.compile(r"\w") + + def word_detokenize( segments: list[list[str]] | list[str], output: str = "str" @@ -346,19 +351,18 @@ def map_indices_to_words(index_list, sentences): for sentence in sentences: words = sentence sentence_result = [] - n = 0 - for start, end in list(c): # Create a snapshot to iterate + # Process elements that belong to this sentence + while c: + start, end = c[0] # Peek at first element if start > n_sum + len(words) - 1: break else: + c.popleft() # Remove after checking word = sentence[start - n_sum : end + 1 - n_sum] sentence_result.append(word) - n += 1 result.append(sentence_result) n_sum += len(words) - for _ in range(n): - c.popleft() # O(1) operation instead of del c[0] return result @@ -465,8 +469,8 @@ def sent_tokenize( result = [] _temp: list[str] = [] for i, w in enumerate(text): - # Use re.search instead of re.findall for boolean checks (faster) - if re.search(r" ", w) and not re.search(r"\w", w): + # Use compiled regex for better performance + if " " in w and not _RE_WORD_CHAR.search(w): if not _temp: continue result.append(_temp) @@ -482,8 +486,8 @@ def sent_tokenize( result = [] _temp = [] for i, w in enumerate(text): - # Use re.search instead of re.findall for boolean checks (faster) - if (re.search(r"\s", w) or re.search(r"\n", w)) and not re.search(r"\w", w): + # Use compiled regex and simplified condition (\\n is included in \\s) + if _RE_WHITESPACE.search(w) and not _RE_WORD_CHAR.search(w): if not _temp: continue result.append(_temp) From ffbfc4e3c9ddd78a1e9f16bbc8059bb616ebc592 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 01:47:01 +0000 Subject: [PATCH 08/15] Remove verbose comments that state the obvious Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/ancient/aksonhan.py | 2 +- pythainlp/corpus/oscar.py | 4 ++-- pythainlp/tokenize/core.py | 10 +++------- pythainlp/wsd/core.py | 2 +- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/pythainlp/ancient/aksonhan.py b/pythainlp/ancient/aksonhan.py index 6786c3ae3..4c9239d34 100644 --- a/pythainlp/ancient/aksonhan.py +++ b/pythainlp/ancient/aksonhan.py @@ -19,7 +19,7 @@ _dict_aksonhan[i + i + j + i] = i + "ั" + j + i _dict_aksonhan[i + i] = "ั" + i _set_aksonhan = set(_dict_aksonhan.keys()) -_trie = Trie(chain(_dict_aksonhan.keys(), thai_consonants)) # Use chain instead of list concatenation +_trie = Trie(chain(_dict_aksonhan.keys(), thai_consonants)) _tokenizer = Tokenizer(custom_dict=_trie, engine="mm") _dict_thai = set(thai_orst_words()) # call Thai words diff --git a/pythainlp/corpus/oscar.py b/pythainlp/corpus/oscar.py index b13718471..f771dcfb2 100644 --- a/pythainlp/corpus/oscar.py +++ b/pythainlp/corpus/oscar.py @@ -30,7 +30,7 @@ def word_freqs() -> list[tuple[str, int]]: path = str(path) with open(path, encoding="utf-8-sig") as f: - lines = f.readlines()[1:] # Skip first line efficiently + lines = f.readlines()[1:] for line in lines: temp = line.strip().split(",") if len(temp) >= 2: @@ -53,7 +53,7 @@ def unigram_word_freqs() -> dict[str, int]: path = str(path) with open(path, encoding="utf-8-sig") as fh: - lines = fh.readlines()[1:] # Skip first line efficiently + lines = fh.readlines()[1:] for i in lines: temp = i.strip().split(",") if temp[0] != " " and '"' not in temp[0]: diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index 4a3e94383..22d9d71ce 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -27,7 +27,6 @@ ) from pythainlp.util.trie import Trie, dict_trie -# Compiled regex patterns for better performance _RE_WHITESPACE = re.compile(r"\s") _RE_WORD_CHAR = re.compile(r"\w") @@ -346,18 +345,17 @@ def indices_words(words): def map_indices_to_words(index_list, sentences): result = [] - c = deque(index_list) # Use deque for O(1) popleft + c = deque(index_list) n_sum = 0 for sentence in sentences: words = sentence sentence_result = [] - # Process elements that belong to this sentence while c: - start, end = c[0] # Peek at first element + start, end = c[0] if start > n_sum + len(words) - 1: break else: - c.popleft() # Remove after checking + c.popleft() word = sentence[start - n_sum : end + 1 - n_sum] sentence_result.append(word) @@ -469,7 +467,6 @@ def sent_tokenize( result = [] _temp: list[str] = [] for i, w in enumerate(text): - # Use compiled regex for better performance if " " in w and not _RE_WORD_CHAR.search(w): if not _temp: continue @@ -486,7 +483,6 @@ def sent_tokenize( result = [] _temp = [] for i, w in enumerate(text): - # Use compiled regex and simplified condition (\\n is included in \\s) if _RE_WHITESPACE.search(w) and not _RE_WORD_CHAR.search(w): if not _temp: continue diff --git a/pythainlp/wsd/core.py b/pythainlp/wsd/core.py index 98f0be8c8..069313f2f 100644 --- a/pythainlp/wsd/core.py +++ b/pythainlp/wsd/core.py @@ -13,7 +13,7 @@ for i, j in zip(_wsd_dict["word"], _wsd_dict["meaning"]): _mean_all[i] = j -_all_word = set(_mean_all.keys()) # dict.keys() is already iterable, no need for list() +_all_word = set(_mean_all.keys()) _TRIE = Trie(_all_word) _word_cut = Tokenizer(custom_dict=_TRIE) From 67260e8105e9699f06320dc3d1bed953f88a6b59 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sun, 11 Jan 2026 02:16:35 +0000 Subject: [PATCH 09/15] Fix type hints --- .python-version | 2 +- pythainlp/spell/pn.py | 27 ++++++++++++++++++--------- pythainlp/tokenize/thaisumcut.py | 26 ++++++++++---------------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/.python-version b/.python-version index e4fba2183..bd28b9c5c 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.12 +3.9 diff --git a/pythainlp/spell/pn.py b/pythainlp/spell/pn.py index 5f8077dc0..450c6f4c5 100644 --- a/pythainlp/spell/pn.py +++ b/pythainlp/spell/pn.py @@ -11,6 +11,7 @@ from collections import Counter from collections.abc import Callable, ItemsView, Iterable from string import digits +from typing import cast from pythainlp import thai_digits, thai_letters from pythainlp.corpus import tnc @@ -35,7 +36,7 @@ def _keep( min_freq: int, min_len: int, max_len: int, - dict_filter: Callable[[str], bool], + dict_filter: Callable[[str], bool] | None, ) -> bool: """ Checks whether a given word has the required minimum frequency min_freq @@ -48,6 +49,9 @@ def _keep( if not (word and min_len <= len(word) <= max_len and word[0] != "."): return False + if not dict_filter: + dict_filter = _no_filter + return dict_filter(word) @@ -60,13 +64,13 @@ def _edits1(word: str) -> set[str]: transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R) > 1] replaces = [L + c + R[1:] for L, R in splits if R for c in thai_letters] inserts = [L + c + R for L, R in splits for c in thai_letters] - + # Use set union for better performance than list concatenation result = set(deletes) result.update(transposes) result.update(replaces) result.update(inserts) - + return result @@ -92,17 +96,22 @@ def _convert_custom_dict( i = iter(custom_dict) first_member = next(i) + + result: list[tuple[str, int]] + if isinstance(first_member, str): # create tuples of a word with frequency equaling 1, # and filter word list - custom_dict = [ + custom_dict = cast(Iterable[str], custom_dict) + result = [ (word, 1) for word in custom_dict if _keep((word, 1), 1, min_len, max_len, dict_filter) ] elif isinstance(first_member, tuple): # filter word list - custom_dict = [ + custom_dict = cast(Iterable[tuple[str, int]], custom_dict) + result = [ word_freq for word_freq in custom_dict if _keep(word_freq, min_freq, min_len, max_len, dict_filter) @@ -113,15 +122,15 @@ def _convert_custom_dict( "Iterable[Tuple[str, int]], or Iterable[str]" ) - return custom_dict + return result class NorvigSpellChecker: def __init__( self, - custom_dict: dict[str, int] - | Iterable[str] - | Iterable[tuple[str, int]] = None, + custom_dict: ( + dict[str, int] | Iterable[str] | Iterable[tuple[str, int]] | None + ) = None, min_freq: int = 2, min_len: int = 2, max_len: int = 40, diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 4113d51fc..3ff2c54a5 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -33,7 +33,7 @@ def middle_cut(sentences: list[str]) -> list[str]: result_parts = [] for sentence in sentences: sentence_size = len(word_tokenize(sentence, keep_whitespace=False)) - + sentence_len = len(sentence) for k in range(0, sentence_len): if k == 0 or k + 1 >= sentence_len: @@ -64,9 +64,7 @@ def middle_cut(sentences: list[str]) -> list[str]: white_space_diff[white_space] = abs(white_space - middle_space) if white_space_diff: - min_diff = min( - white_space_diff.items(), key=operator.itemgetter(1) - ) + min_diff = min(white_space_diff.items(), key=operator.itemgetter(1)) tokens.pop(min_diff[0]) tokens.insert(min_diff[0], "") result_parts.append(list_to_string(tokens)) @@ -77,20 +75,20 @@ def middle_cut(sentences: list[str]) -> list[str]: all_sentences = [] for part in result_parts: all_sentences.extend(part.split("")) - + sentences = [s.strip() for s in all_sentences if s.strip()] - + return sentences class ThaiSentenceSegmentor: - def split_into_sentences( - self, text: str, isMiddleCut: bool = False - ) -> list[str]: + def split_into_sentences(self, text: str, isMiddleCut: bool = False) -> list[str]: # Declare Variables th_alphabets = "([ก-๙])" th_conjunction = "(ทำให้|โดย|เพราะ|นอกจากนี้|แต่|กรณีที่|หลังจากนี้|ต่อมา|ภายหลัง|นับตั้งแต่|หลังจาก|ซึ่งเหตุการณ์|ผู้สื่อข่าวรายงานอีก|ส่วนที่|ส่วนสาเหตุ|ฉะนั้น|เพราะฉะนั้น|เพื่อ|เนื่องจาก|จากการสอบสวนทราบว่า|จากกรณี|จากนี้|อย่างไรก็ดี)" - th_cite = "(กล่าวว่า|เปิดเผยว่า|รายงานว่า|ให้การว่า|เผยว่า|บนทวิตเตอร์ว่า|แจ้งว่า|พลเมืองดีว่า|อ้างว่า)" + th_cite = ( + "(กล่าวว่า|เปิดเผยว่า|รายงานว่า|ให้การว่า|เผยว่า|บนทวิตเตอร์ว่า|แจ้งว่า|พลเมืองดีว่า|อ้างว่า)" + ) th_ka_krub = "(ครับ|ค่ะ)" th_stop_after = "(หรือไม่|โดยเร็ว|แล้ว|อีกด้วย)" th_stop_before = "(ล่าสุด|เบื้องต้น|ซึ่ง|ทั้งนี้|แม้ว่า|เมื่อ|แถมยัง|ตอนนั้น|จนเป็นเหตุให้|จากนั้น|อย่างไรก็ตาม|และก็|อย่างใดก็ตาม|เวลานี้|เช่น|กระทั่ง)" @@ -156,9 +154,7 @@ def split_into_sentences( text = text.replace("ทั้งนี้เพื่อ", "ทั้งนี้") text = text.replace("เวลาต่อมา", "เวลา") text = text.replace("อย่างไรก็ตาม", "อย่างไรก็ตาม") - text = text.replace( - "อย่างไรก็ตามหลังจาก", "อย่างไรก็ตาม" - ) + text = text.replace("อย่างไรก็ตามหลังจาก", "อย่างไรก็ตาม") text = text.replace("ซึ่งทำให้", "ซึ่ง") text = text.replace("โดยประมาท", "ประมาท") text = text.replace("โดยธรรม", "ธรรม") @@ -287,9 +283,7 @@ def split_into_sentences( text = re.sub(th_conjunction, "\\1", text) text = re.sub(th_cite, "\\1", text) text = re.sub(" " + degit + "[.]" + th_title, "\\1.\\2", text) - text = re.sub( - " " + degit + degit + "[.]" + th_title, "\\1\\2.\\3", text - ) + text = re.sub(" " + degit + degit + "[.]" + th_title, "\\1\\2.\\3", text) text = re.sub(th_alphabets + th_stop_after + " ", "\\1\\2", text) if "”" in text: text = text.replace(".”", "”.") From 97ebfa3478b6f130f7e88d2ca054d905b36978f9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 02:37:20 +0000 Subject: [PATCH 10/15] Optimize corpus reading: iterate directly without loading all lines into memory Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/corpus/oscar.py | 10 +++++----- pythainlp/corpus/tnc.py | 8 +++----- pythainlp/corpus/ttc.py | 8 +++----- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/pythainlp/corpus/oscar.py b/pythainlp/corpus/oscar.py index f771dcfb2..0fe0b8c67 100644 --- a/pythainlp/corpus/oscar.py +++ b/pythainlp/corpus/oscar.py @@ -30,8 +30,8 @@ def word_freqs() -> list[tuple[str, int]]: path = str(path) with open(path, encoding="utf-8-sig") as f: - lines = f.readlines()[1:] - for line in lines: + next(f) # Skip header line + for line in f: temp = line.strip().split(",") if len(temp) >= 2: if temp[0] != " " and '"' not in temp[0]: @@ -53,9 +53,9 @@ def unigram_word_freqs() -> dict[str, int]: path = str(path) with open(path, encoding="utf-8-sig") as fh: - lines = fh.readlines()[1:] - for i in lines: - temp = i.strip().split(",") + next(fh) # Skip header line + for line in fh: + temp = line.strip().split(",") if temp[0] != " " and '"' not in temp[0]: freqs[temp[0]] = int(temp[-1]) elif temp[0] == " ": diff --git a/pythainlp/corpus/tnc.py b/pythainlp/corpus/tnc.py index 7aef7b5c2..a9c80e6e9 100644 --- a/pythainlp/corpus/tnc.py +++ b/pythainlp/corpus/tnc.py @@ -31,8 +31,7 @@ def word_freqs() -> list[tuple[str, int]]: Credit: Korakot Chaovavanich https://www.facebook.com/groups/thainlp/posts/434330506948445 """ freqs: list[tuple[str, int]] = [] - lines = list(get_corpus(_UNIGRAM_FILENAME)) - for line in lines: + for line in get_corpus(_UNIGRAM_FILENAME): word_freq = line.split("\t") if len(word_freq) >= 2: freqs.append((word_freq[0], int(word_freq[1]))) @@ -45,9 +44,8 @@ def unigram_word_freqs() -> dict[str, int]: Get unigram word frequency from Thai National Corpus (TNC) """ freqs: dict[str, int] = defaultdict(int) - lines = list(get_corpus(_UNIGRAM_FILENAME)) - for i in lines: - _temp = i.strip().split(" ") + for line in get_corpus(_UNIGRAM_FILENAME): + _temp = line.strip().split(" ") if len(_temp) >= 2: freqs[_temp[0]] = int(_temp[-1]) diff --git a/pythainlp/corpus/ttc.py b/pythainlp/corpus/ttc.py index a32051259..95e6ae5da 100644 --- a/pythainlp/corpus/ttc.py +++ b/pythainlp/corpus/ttc.py @@ -26,8 +26,7 @@ def word_freqs() -> list[tuple[str, int]]: `_) """ freqs: list[tuple[str, int]] = [] - lines = list(get_corpus(_UNIGRAM_FILENAME)) - for line in lines: + for line in get_corpus(_UNIGRAM_FILENAME): word_freq = line.split("\t") if len(word_freq) >= 2: freqs.append((word_freq[0], int(word_freq[1]))) @@ -41,9 +40,8 @@ def unigram_word_freqs() -> dict[str, int]: """ freqs: dict[str, int] = defaultdict(int) - lines = list(get_corpus(_UNIGRAM_FILENAME)) - for i in lines: - temp = i.strip().split(" ") + for line in get_corpus(_UNIGRAM_FILENAME): + temp = line.strip().split(" ") if len(temp) >= 2: freqs[temp[0]] = int(temp[-1]) From 26824e6683a48c929d6cf20f1ca5e0696bdb6414 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 02:39:12 +0000 Subject: [PATCH 11/15] Use list comprehensions and generator expressions for more efficient iteration Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/tokenize/crfcut.py | 7 ++----- pythainlp/tokenize/thaisumcut.py | 14 +++++++------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/pythainlp/tokenize/crfcut.py b/pythainlp/tokenize/crfcut.py index 3c15d2b6e..0eed223cc 100644 --- a/pythainlp/tokenize/crfcut.py +++ b/pythainlp/tokenize/crfcut.py @@ -148,11 +148,8 @@ def extract_features( doc = padded_doc # add enders and starters - doc_ender = [] - doc_starter = [] - for token in doc: - doc_ender.append("ender" if token in _ENDERS else "normal") - doc_starter.append("starter" if token in _STARTERS else "normal") + doc_ender = ["ender" if token in _ENDERS else "normal" for token in doc] + doc_starter = ["starter" if token in _STARTERS else "normal" for token in doc] # for each word for i in range(window, len(doc) - window): diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 3ff2c54a5..5cc0d7ab5 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -72,13 +72,13 @@ def middle_cut(sentences: list[str]) -> list[str]: result_parts.append(sentence) # Split all result parts by and filter - all_sentences = [] - for part in result_parts: - all_sentences.extend(part.split("")) - - sentences = [s.strip() for s in all_sentences if s.strip()] - - return sentences + all_sentences = ( + s.strip() + for part in result_parts + for s in part.split("") + ) + + return [s for s in all_sentences if s] class ThaiSentenceSegmentor: From 27d1408b42f791fd6a7ca4ec6af917a00ed10ae1 Mon Sep 17 00:00:00 2001 From: Arthit Suriyawongkul Date: Sun, 11 Jan 2026 02:44:51 +0000 Subject: [PATCH 12/15] format --- pythainlp/tokenize/thaisumcut.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 5cc0d7ab5..a2f473d59 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -72,12 +72,8 @@ def middle_cut(sentences: list[str]) -> list[str]: result_parts.append(sentence) # Split all result parts by and filter - all_sentences = ( - s.strip() - for part in result_parts - for s in part.split("") - ) - + all_sentences = (s.strip() for part in result_parts for s in part.split("")) + return [s for s in all_sentences if s] From 52a8f4b46c8280cc0a73fe0bd4cbe36847a952f8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 03:37:23 +0000 Subject: [PATCH 13/15] Add early returns to avoid unnecessary computation Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/corpus/tnc.py | 14 ++++++++------ pythainlp/lm/text_util.py | 2 ++ pythainlp/tokenize/crfcut.py | 3 +++ pythainlp/tokenize/thaisumcut.py | 3 +++ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/pythainlp/corpus/tnc.py b/pythainlp/corpus/tnc.py index a9c80e6e9..2bf203cef 100644 --- a/pythainlp/corpus/tnc.py +++ b/pythainlp/corpus/tnc.py @@ -63,9 +63,10 @@ def bigram_word_freqs() -> dict[tuple[str, str], int]: path = str(path) with open(path, encoding="utf-8-sig") as fh: - for i in fh.readlines(): - temp = i.strip().split(" ") - freqs[(temp[0], temp[1])] = int(temp[-1]) + for line in fh: + temp = line.strip().split(" ") + if len(temp) >= 3: + freqs[(temp[0], temp[1])] = int(temp[-1]) return freqs @@ -81,8 +82,9 @@ def trigram_word_freqs() -> dict[tuple[str, str, str], int]: path = str(path) with open(path, encoding="utf-8-sig") as fh: - for i in fh.readlines(): - temp = i.strip().split(" ") - freqs[(temp[0], temp[1], temp[2])] = int(temp[-1]) + for line in fh: + temp = line.strip().split(" ") + if len(temp) >= 4: + freqs[(temp[0], temp[1], temp[2])] = int(temp[-1]) return freqs diff --git a/pythainlp/lm/text_util.py b/pythainlp/lm/text_util.py index 35e4b9167..24b606409 100644 --- a/pythainlp/lm/text_util.py +++ b/pythainlp/lm/text_util.py @@ -18,6 +18,8 @@ def calculate_ngram_counts( :return: A dictionary where keys are n-grams and values are their counts. :rtype: Dict[Tuple[str], int] """ + if not list_words: + return {} ngram_counts = {} diff --git a/pythainlp/tokenize/crfcut.py b/pythainlp/tokenize/crfcut.py index 0eed223cc..a53bee7a1 100644 --- a/pythainlp/tokenize/crfcut.py +++ b/pythainlp/tokenize/crfcut.py @@ -140,6 +140,9 @@ def extract_features( within the `window` :return: list of lists of features to be fed to CRF """ + if not doc: + return [] + doc_features = [] # Pad the document with "xxpad" tokens efficiently padded_doc = ["xxpad"] * window diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index a2f473d59..697ef04f3 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -30,6 +30,9 @@ def list_to_string(list: list[str]) -> str: def middle_cut(sentences: list[str]) -> list[str]: + if not sentences: + return [] + result_parts = [] for sentence in sentences: sentence_size = len(word_tokenize(sentence, keep_whitespace=False)) From 1b004acf62c01962e57b46fcce6c1675fdf16d95 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 13:29:32 +0000 Subject: [PATCH 14/15] Fix Ruff issues: remove unnecessary map(), list() calls, use dict comprehensions, remove unused import, fix default parameter, and remove trailing whitespace Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/benchmarks/word_tokenization.py | 2 +- pythainlp/cli/benchmark.py | 2 +- pythainlp/corpus/common.py | 2 +- pythainlp/morpheme/word_formation.py | 2 +- pythainlp/tag/blackboard.py | 2 +- pythainlp/tag/orchid.py | 2 +- pythainlp/tokenize/core.py | 1 - pythainlp/tokenize/crfcut.py | 2 +- pythainlp/tokenize/thaisumcut.py | 2 +- pythainlp/transliterate/thai2rom_onnx.py | 6 +++--- pythainlp/wsd/core.py | 2 +- 11 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index 95cda14bf..fc2eaa2c3 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -257,5 +257,5 @@ def _find_words_correctly_tokenised( """ ref_b = dict(zip(ref_boundaries, [1] * len(ref_boundaries))) - labels = tuple(map(lambda x: ref_b.get(x, 0), predicted_boundaries)) + labels = tuple(ref_b.get(x, 0) for x in predicted_boundaries) return labels diff --git a/pythainlp/cli/benchmark.py b/pythainlp/cli/benchmark.py index e342ff9fd..f85f72fa4 100644 --- a/pythainlp/cli/benchmark.py +++ b/pythainlp/cli/benchmark.py @@ -13,7 +13,7 @@ def _read_file(path): with open(path, encoding="utf-8") as f: - lines = map(lambda r: r.strip(), f.readlines()) + lines = (r.strip() for r in f.readlines()) return list(lines) diff --git a/pythainlp/corpus/common.py b/pythainlp/corpus/common.py index cf467a754..044c063f6 100644 --- a/pythainlp/corpus/common.py +++ b/pythainlp/corpus/common.py @@ -372,7 +372,7 @@ def find_synonyms(word: str) -> list[str]: list_synonym.extend(synonyms["synonym"][idx]) list_synonym.append(synonyms["word"][idx]) - list_synonym = sorted(list(set(list_synonym))) + list_synonym = sorted(set(list_synonym)) if word in list_synonym: # remove same word list_synonym.remove(word) diff --git a/pythainlp/morpheme/word_formation.py b/pythainlp/morpheme/word_formation.py index 212c3e23a..63453a25e 100644 --- a/pythainlp/morpheme/word_formation.py +++ b/pythainlp/morpheme/word_formation.py @@ -35,7 +35,7 @@ def nighit(w1: str, w2: str) -> str: raise NotImplementedError(f"The function doesn't support {w1}.") list_w1 = list(w1) list_w2 = list(w2) - newword = list() + newword = [] newword.append(list_w1[0]) newword.append("ั") consonant_start = [i for i in list_w2 if i in set(thai_consonants)][0] diff --git a/pythainlp/tag/blackboard.py b/pythainlp/tag/blackboard.py index 2a3938f14..42c43e2c3 100644 --- a/pythainlp/tag/blackboard.py +++ b/pythainlp/tag/blackboard.py @@ -5,7 +5,7 @@ # defined strings for special characters CHAR_TO_ESCAPE = {" ": "_"} -ESCAPE_TO_CHAR = dict((v, k) for k, v in CHAR_TO_ESCAPE.items()) +ESCAPE_TO_CHAR = {v: k for k, v in CHAR_TO_ESCAPE.items()} # map from Blackboard treebank POS tag to Universal POS tag diff --git a/pythainlp/tag/orchid.py b/pythainlp/tag/orchid.py index 69fcb0c07..f9a905a57 100644 --- a/pythainlp/tag/orchid.py +++ b/pythainlp/tag/orchid.py @@ -33,7 +33,7 @@ ";": "", "/": "", } -ESCAPE_TO_CHAR = dict((v, k) for k, v in CHAR_TO_ESCAPE.items()) +ESCAPE_TO_CHAR = {v: k for k, v in CHAR_TO_ESCAPE.items()} # map from ORCHID POS tag to Universal POS tag # from Korakot Chaovavanich diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index 74e9f4158..f7b6d9d10 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -6,7 +6,6 @@ from __future__ import annotations -import copy import re from collections import deque from collections.abc import Iterable diff --git a/pythainlp/tokenize/crfcut.py b/pythainlp/tokenize/crfcut.py index a00833e7e..bbf437b05 100644 --- a/pythainlp/tokenize/crfcut.py +++ b/pythainlp/tokenize/crfcut.py @@ -140,7 +140,7 @@ def extract_features( """ if not doc: return [] - + doc_features = [] # Pad the document with "xxpad" tokens efficiently padded_doc = ["xxpad"] * window diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 9665f9f6c..9909c0c30 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -31,7 +31,7 @@ def list_to_string(list: list[str]) -> str: def middle_cut(sentences: list[str]) -> list[str]: if not sentences: return [] - + result_parts = [] for sentence in sentences: sentence_size = len(word_tokenize(sentence, keep_whitespace=False)) diff --git a/pythainlp/transliterate/thai2rom_onnx.py b/pythainlp/transliterate/thai2rom_onnx.py index 19ebb0468..ab00131d2 100644 --- a/pythainlp/transliterate/thai2rom_onnx.py +++ b/pythainlp/transliterate/thai2rom_onnx.py @@ -127,9 +127,9 @@ def run(self, source_seq, source_seq_len): outputs = np.zeros((max_len, batch_size, self.target_vocab_size)) - expected_encoder_outputs = list( - map(lambda output: output.name, self.encoder.get_outputs()) - ) + expected_encoder_outputs = [ + output.name for output in self.encoder.get_outputs() + ] encoder_outputs, encoder_hidden, _ = self.encoder.run( input_feed={ "input_tensor": source_seq, diff --git a/pythainlp/wsd/core.py b/pythainlp/wsd/core.py index 16ff005d0..669b74dcb 100644 --- a/pythainlp/wsd/core.py +++ b/pythainlp/wsd/core.py @@ -50,7 +50,7 @@ def get_sense( sentence: str, word: str, device: str = "cpu", - custom_dict: dict = dict(), + custom_dict: dict | None = None, custom_tokenizer: Tokenizer = _word_cut, ) -> list[tuple[str, float]]: """Get word sense from the sentence. From 08092b268c8868757eea4db1a45d924efa0c1bbb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 11 Jan 2026 13:42:54 +0000 Subject: [PATCH 15/15] Fix ResourceWarning: add exception handling for file operations in tnc.py Co-authored-by: bact <128572+bact@users.noreply.github.com> --- pythainlp/corpus/tnc.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/pythainlp/corpus/tnc.py b/pythainlp/corpus/tnc.py index 3371452e6..7a01849a6 100644 --- a/pythainlp/corpus/tnc.py +++ b/pythainlp/corpus/tnc.py @@ -58,11 +58,14 @@ def bigram_word_freqs() -> dict[tuple[str, str], int]: return freqs path = str(path) - with open(path, encoding="utf-8-sig") as fh: - for line in fh: - temp = line.strip().split(" ") - if len(temp) >= 3: - freqs[(temp[0], temp[1])] = int(temp[-1]) + try: + with open(path, encoding="utf-8-sig") as fh: + for line in fh: + temp = line.strip().split(" ") + if len(temp) >= 3: + freqs[(temp[0], temp[1])] = int(temp[-1]) + except (IOError, OSError): + pass return freqs @@ -76,10 +79,13 @@ def trigram_word_freqs() -> dict[tuple[str, str, str], int]: return freqs path = str(path) - with open(path, encoding="utf-8-sig") as fh: - for line in fh: - temp = line.strip().split(" ") - if len(temp) >= 4: - freqs[(temp[0], temp[1], temp[2])] = int(temp[-1]) + try: + with open(path, encoding="utf-8-sig") as fh: + for line in fh: + temp = line.strip().split(" ") + if len(temp) >= 4: + freqs[(temp[0], temp[1], temp[2])] = int(temp[-1]) + except (IOError, OSError): + pass return freqs