diff --git a/.python-version b/.python-version index e4fba2183..bd28b9c5c 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.12 +3.9 diff --git a/pythainlp/ancient/aksonhan.py b/pythainlp/ancient/aksonhan.py index e9fef0cec..67acaae21 100644 --- a/pythainlp/ancient/aksonhan.py +++ b/pythainlp/ancient/aksonhan.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from itertools import chain + from pythainlp import thai_consonants, thai_tonemarks from pythainlp.corpus import thai_orst_words from pythainlp.tokenize import Tokenizer @@ -17,7 +19,7 @@ _dict_aksonhan[i + i + j + i] = i + "ั" + j + i _dict_aksonhan[i + i] = "ั" + i _set_aksonhan = set(_dict_aksonhan.keys()) -_trie = Trie(list(_dict_aksonhan.keys()) + list(thai_consonants)) +_trie = Trie(chain(_dict_aksonhan.keys(), thai_consonants)) _tokenizer = Tokenizer(custom_dict=_trie, engine="mm") _dict_thai = set(thai_orst_words()) # call Thai words diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index 94e7598f3..fc2eaa2c3 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -56,13 +56,11 @@ def _flatten_result(my_dict: dict, sep: str = ":") -> dict: :return: a one-dimension dictionary with keys combined :rtype: dict[str, float | str] """ - items = [] - for k1, kv2 in my_dict.items(): - for k2, v in kv2.items(): - new_key = f"{k1}{sep}{k2}" - items.append((new_key, v)) - - return dict(items) + return { + f"{k1}{sep}{k2}": v + for k1, kv2 in my_dict.items() + for k2, v in kv2.items() + } def benchmark(ref_samples: list[str], samples: list[str]) -> pd.DataFrame: @@ -259,5 +257,5 @@ def _find_words_correctly_tokenised( """ ref_b = dict(zip(ref_boundaries, [1] * len(ref_boundaries))) - labels = tuple(map(lambda x: ref_b.get(x, 0), predicted_boundaries)) + labels = tuple(ref_b.get(x, 0) for x in predicted_boundaries) return labels diff --git a/pythainlp/cli/benchmark.py b/pythainlp/cli/benchmark.py index e342ff9fd..f85f72fa4 100644 --- a/pythainlp/cli/benchmark.py +++ b/pythainlp/cli/benchmark.py @@ -13,7 +13,7 @@ def _read_file(path): with open(path, encoding="utf-8") as f: - lines = map(lambda r: r.strip(), f.readlines()) + lines = (r.strip() for r in f.readlines()) return list(lines) diff --git a/pythainlp/corpus/common.py b/pythainlp/corpus/common.py index cf467a754..044c063f6 100644 --- a/pythainlp/corpus/common.py +++ b/pythainlp/corpus/common.py @@ -372,7 +372,7 @@ def find_synonyms(word: str) -> list[str]: list_synonym.extend(synonyms["synonym"][idx]) list_synonym.append(synonyms["word"][idx]) - list_synonym = sorted(list(set(list_synonym))) + list_synonym = sorted(set(list_synonym)) if word in list_synonym: # remove same word list_synonym.remove(word) diff --git a/pythainlp/corpus/core.py b/pythainlp/corpus/core.py index f509a0f02..689c28bd0 100644 --- a/pythainlp/corpus/core.py +++ b/pythainlp/corpus/core.py @@ -188,8 +188,8 @@ def get_corpus_default_db(name: str, version: str = "") -> str | None: with open(default_db_path, encoding="utf-8-sig") as fh: corpus_db = json.load(fh) - if name in list(corpus_db.keys()): - if version in list(corpus_db[name]["versions"].keys()): + if name in corpus_db: + if version in corpus_db[name]["versions"]: return path_pythainlp_corpus( corpus_db[name]["versions"][version]["filename"] ) @@ -247,7 +247,7 @@ def get_corpus_path( CUSTOMIZE: dict[str, str] = { # "the corpus name":"path" } - if name in list(CUSTOMIZE): + if name in CUSTOMIZE: return CUSTOMIZE[name] default_path = get_corpus_default_db(name=name, version=version) diff --git a/pythainlp/corpus/oscar.py b/pythainlp/corpus/oscar.py index 193af0e2c..d9696e488 100644 --- a/pythainlp/corpus/oscar.py +++ b/pythainlp/corpus/oscar.py @@ -28,9 +28,8 @@ def word_freqs() -> list[tuple[str, int]]: path = str(path) with open(path, encoding="utf-8-sig") as f: - lines = list(f.readlines()) - del lines[0] - for line in lines: + next(f) # Skip header line + for line in f: temp = line.strip().split(",") if len(temp) >= 2: if temp[0] != " " and '"' not in temp[0]: @@ -51,10 +50,9 @@ def unigram_word_freqs() -> dict[str, int]: path = str(path) with open(path, encoding="utf-8-sig") as fh: - lines = list(fh.readlines()) - del lines[0] - for i in lines: - temp = i.strip().split(",") + next(fh) # Skip header line + for line in fh: + temp = line.strip().split(",") if temp[0] != " " and '"' not in temp[0]: freqs[temp[0]] = int(temp[-1]) elif temp[0] == " ": diff --git a/pythainlp/corpus/tnc.py b/pythainlp/corpus/tnc.py index af9513c6d..7a01849a6 100644 --- a/pythainlp/corpus/tnc.py +++ b/pythainlp/corpus/tnc.py @@ -29,8 +29,7 @@ def word_freqs() -> list[tuple[str, int]]: Credit: Korakot Chaovavanich https://www.facebook.com/groups/thainlp/posts/434330506948445 """ freqs: list[tuple[str, int]] = [] - lines = list(get_corpus(_UNIGRAM_FILENAME)) - for line in lines: + for line in get_corpus(_UNIGRAM_FILENAME): word_freq = line.split("\t") if len(word_freq) >= 2: freqs.append((word_freq[0], int(word_freq[1]))) @@ -42,9 +41,8 @@ def unigram_word_freqs() -> dict[str, int]: """Get unigram word frequency from Thai National Corpus (TNC) """ freqs: dict[str, int] = defaultdict(int) - lines = list(get_corpus(_UNIGRAM_FILENAME)) - for i in lines: - _temp = i.strip().split(" ") + for line in get_corpus(_UNIGRAM_FILENAME): + _temp = line.strip().split(" ") if len(_temp) >= 2: freqs[_temp[0]] = int(_temp[-1]) @@ -60,10 +58,14 @@ def bigram_word_freqs() -> dict[tuple[str, str], int]: return freqs path = str(path) - with open(path, encoding="utf-8-sig") as fh: - for i in fh.readlines(): - temp = i.strip().split(" ") - freqs[(temp[0], temp[1])] = int(temp[-1]) + try: + with open(path, encoding="utf-8-sig") as fh: + for line in fh: + temp = line.strip().split(" ") + if len(temp) >= 3: + freqs[(temp[0], temp[1])] = int(temp[-1]) + except (IOError, OSError): + pass return freqs @@ -77,9 +79,13 @@ def trigram_word_freqs() -> dict[tuple[str, str, str], int]: return freqs path = str(path) - with open(path, encoding="utf-8-sig") as fh: - for i in fh.readlines(): - temp = i.strip().split(" ") - freqs[(temp[0], temp[1], temp[2])] = int(temp[-1]) + try: + with open(path, encoding="utf-8-sig") as fh: + for line in fh: + temp = line.strip().split(" ") + if len(temp) >= 4: + freqs[(temp[0], temp[1], temp[2])] = int(temp[-1]) + except (IOError, OSError): + pass return freqs diff --git a/pythainlp/corpus/ttc.py b/pythainlp/corpus/ttc.py index 99134ca4b..908d9db74 100644 --- a/pythainlp/corpus/ttc.py +++ b/pythainlp/corpus/ttc.py @@ -24,8 +24,7 @@ def word_freqs() -> list[tuple[str, int]]: `_) """ freqs: list[tuple[str, int]] = [] - lines = list(get_corpus(_UNIGRAM_FILENAME)) - for line in lines: + for line in get_corpus(_UNIGRAM_FILENAME): word_freq = line.split("\t") if len(word_freq) >= 2: freqs.append((word_freq[0], int(word_freq[1]))) @@ -38,9 +37,8 @@ def unigram_word_freqs() -> dict[str, int]: """ freqs: dict[str, int] = defaultdict(int) - lines = list(get_corpus(_UNIGRAM_FILENAME)) - for i in lines: - temp = i.strip().split(" ") + for line in get_corpus(_UNIGRAM_FILENAME): + temp = line.strip().split(" ") if len(temp) >= 2: freqs[temp[0]] = int(temp[-1]) diff --git a/pythainlp/khavee/core.py b/pythainlp/khavee/core.py index 5f8b0281f..83e047263 100644 --- a/pythainlp/khavee/core.py +++ b/pythainlp/khavee/core.py @@ -1,682 +1,689 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 -# ruff: noqa: C901 -from __future__ import annotations - -from pythainlp import thai_consonants -from pythainlp.tokenize import subword_tokenize -from pythainlp.util import remove_tonemark, sound_syllable - - -class KhaveeVerifier: - def __init__(self): - """KhaveeVerifier: Thai Poetry verifier - """ - - def _has_true_final_yl(self, word: str) -> bool: - """Check if ย or ล is a true final consonant - (not just part of the vowel sound with ไ/ใ) - - :param str word: Thai word - :return: True if ย or ล is a true final consonant - :rtype: bool - """ - if len(word) < 2: - return False - # Count consonants in the word - consonant_count = sum(1 for c in word if c in thai_consonants) - # If there are 2+ consonants and word ends with ย or ล, it's a true final - return consonant_count >= 2 and word[-1] in ["ย", "ล"] - - def check_sara(self, word: str) -> str: - """Check the vowels in the Thai word. - - :param str word: Thai word - :return: vowel name of the word - :rtype: str - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.check_sara("เริง")) - # output: 'เออ' - """ - sara = [] - countoa = 0 - - # In case of การันย์ - if "์" in word[-1]: - word = word[:-2] - - # In case of สระเดี่ยว - for i in word: - if i in ("ะ", "ั"): - sara.append("อะ") - elif i == "ิ": - sara.append("อิ") - elif i == "ุ": - sara.append("อุ") - elif i == "ึ": - sara.append("อึ") - elif i == "ี": - sara.append("อี") - elif i == "ู": - sara.append("อู") - elif i == "ื": - sara.append("อือ") - elif i == "เ": - sara.append("เอ") - elif i == "แ": - sara.append("แอ") - elif i == "า": - sara.append("อา") - elif i == "โ": - sara.append("โอ") - elif i == "ำ": - sara.append("อำ") - elif i == "อ": - countoa += 1 - sara.append("ออ") - elif i == "ั" and "ว" in word: - sara.append("อัว") - elif i in ("ไ", "ใ"): - sara.append("ไอ") - elif i == "็": - sara.append("ออ") - elif "รร" in word: - if self.check_marttra(word) == "กม": - sara.append("อำ") - else: - sara.append("อะ") - - # In case of ออ - if countoa == 1 and "อ" in word[-1] and "เ" not in word: - sara.remove("ออ") - - # In case of เอ เอ - countA = 0 - for i in sara: - if i == "เอ": - countA = countA + 1 - if countA > 1: - sara.remove("เอ") - sara.remove("เอ") - sara.append("แ") - - # In case of สระประสม - if "เอ" in sara and "อะ" in sara: - sara.remove("เอ") - sara.remove("อะ") - sara.append("เอะ") - elif "แอ" in sara and "อะ" in sara: - sara.remove("แอ") - sara.remove("อะ") - sara.append("แอะ") - - if "เอะ" in sara and "ออ" in sara: - sara.remove("เอะ") - sara.remove("ออ") - sara.append("เออะ") - elif "เอ" in sara and "อิ" in sara: - sara.remove("เอ") - sara.remove("อิ") - sara.append("เออ") - elif "เอ" in sara and "ออ" in sara and "อ" in word[-1]: - sara.remove("เอ") - sara.remove("ออ") - sara.append("เออ") - elif "โอ" in sara and "อะ" in sara: - sara.remove("โอ") - sara.remove("อะ") - sara.append("โอะ") - elif "เอ" in sara and "อี" in sara: - sara.remove("เอ") - sara.remove("อี") - sara.append("เอีย") - elif "เอ" in sara and "อือ" in sara: - sara.remove("เอ") - sara.remove("อือ") - sara.append("อัว") - elif "เอ" in sara and "อา" in sara: - sara.remove("เอ") - sara.remove("อา") - sara.append("เอา") - elif "เ" in word and "า" in word and "ะ" in word: - sara = [] - sara.append("เอาะ") - - if "อือ" in sara and "เออ" in sara: - sara.remove("เออ") - sara.remove("อือ") - sara.append("เอือ") - elif "ออ" in sara and len(sara) > 1: - sara.remove("ออ") - elif "ว" in word and len(sara) == 0: - sara.append("อัว") - - if "ั" in word and self.check_marttra(word) == "กา": - sara = [] - sara.append("ไอ") - - # In case of อ - if word == "เออะ": - sara = [] - sara.append("เออะ") - elif word == "เออ": - sara = [] - sara.append("เออ") - elif word == "เอ": - sara = [] - sara.append("เอ") - elif word == "เอะ": - sara = [] - sara.append("เอะ") - elif word == "เอา": - sara = [] - sara.append("เอา") - elif word == "เอาะ": - sara = [] - sara.append("เอาะ") - - if "ฤา" in word or "ฦา" in word: - sara = [] - sara.append("อือ") - elif "ฤ" in word or "ฦ" in word: - sara = [] - sara.append("อึ") - - # In case of กน - if not sara and len(word) == 2: - if word[-1] != "ร": - sara.append("โอะ") - else: - sara.append("ออ") - elif not sara and len(word) == 3: - sara.append("ออ") - - # In case of บ่ - if word == "บ่": - sara = [] - sara.append("ออ") - - if "ํ" in word: - sara = [] - sara.append("อำ") - - if "เ" in word and "ื" in word and "อ" in word: - sara = [] - sara.append("เอือ") - - if not sara: - return "Can't find Sara in this word" - - return sara[0] - - def check_marttra(self, word: str) -> str: - """Check the Thai spelling Section in the Thai word. - - :param str word: Thai word - :return: name of spelling Section of the word. - :rtype: str - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.check_marttra("สาว")) - # output: 'เกอว' - """ - # Handle consonant clusters ending with ร - # ตร, ทร → remove ร (treat as final ต/ท sound) - # กร, ขร, คร, ฆร in compound words → remove ร (treat as final ก/ข/ค sound) - # But single syllable words like "กร" should keep ร - if len(word) >= 3 and word[-1] == "ร": - if word[-2] in ["ต", "ท"]: - word = word[:-1] - elif word[-2] in ["ก", "ข", "ค", "ฆ"]: - word = word[:-1] - - word = self.handle_karun_sound_silence(word) - word = remove_tonemark(word) - - # Check for ำ at the end (represents "am" sound, ends with m) - if word[-1] == "ำ": - return "กม" - - # Check for vowels and special patterns that indicate open syllables (กา) - # For words with ไ/ใ, check if ย/ล is a true final or just part of vowel - if "ไ" in word or "ใ" in word: - if word[-1] not in ["ย", "ล"]: - return "กา" - elif not self._has_true_final_yl(word): - # ย/ล is part of the vowel sound, not a true final - return "กา" - # else: ย/ล is a true final, continue to consonant classification below - - if "ํ" in word and "า" in word: - return "กา" - elif ( - word[-1] in ["า", "ะ", "ิ", "ี", "ุ", "ู", "อ"] - or ("ี" in word and "ย" in word[-1]) - or ("ื" in word and "อ" in word[-1]) - ): - return "กา" - elif word[-1] in ["ง"]: - return "กง" - elif word[-1] in ["ม"]: - return "กม" - elif word[-1] in ["ย"]: - return "เกย" - elif word[-1] in ["ล"]: - return "เกย" - elif word[-1] in ["ว"]: - return "เกอว" - elif word[-1] in ["ก", "ข", "ค", "ฆ"]: - return "กก" - elif word[-1] in [ - "จ", - "ช", - "ซ", - "ฎ", - "ฏ", - "ฐ", - "ฑ", - "ฒ", - "ด", - "ต", - "ถ", - "ท", - "ธ", - "ศ", - "ษ", - "ส", - ]: - return "กด" - elif word[-1] in ["ญ", "ณ", "น", "ร", "ฬ"]: - return "กน" - elif word[-1] in ["บ", "ป", "พ", "ฟ", "ภ"]: - return "กบ" - else: - if "็" in word: - return "กา" - else: - return "Cant find Marttra in this word" - - def is_sumpus(self, word1: str, word2: str) -> bool: - """Check the rhyme between two words. - - :param str word1: Thai word - :param str word2: Thai word - :return: boolean - :rtype: bool - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.is_sumpus("สรร", "อัน")) - # output: True - - print(kv.is_sumpus("สรร", "แมว")) - # output: False - """ - marttra1 = self.check_marttra(word1) - marttra2 = self.check_marttra(word2) - sara1 = self.check_sara(word1) - sara2 = self.check_sara(word2) - if sara1 == "อะ" and marttra1 == "เกย": - sara1 = "ไอ" - marttra1 = "กา" - elif sara2 == "อะ" and marttra2 == "เกย": - sara2 = "ไอ" - marttra2 = "กา" - if sara1 == "อำ" and marttra1 == "กม": - sara1 = "อำ" - marttra1 = "กา" - elif sara2 == "อำ" and marttra2 == "กม": - sara2 = "อำ" - marttra2 = "กา" - return bool(marttra1 == marttra2 and sara1 == sara2) - - def check_karu_lahu(self, text): - if ( - self.check_marttra(text) != "กา" - or ( - self.check_marttra(text) == "กา" - and self.check_sara(text) - in [ - "อา", - "อี", - "อือ", - "อู", - "เอ", - "แอ", - "โอ", - "ออ", - "เออ", - "เอีย", - "เอือ", - "อัว", - ] - ) - or self.check_sara(text) in ["อำ", "ไอ", "เอา"] - ) and text not in ["บ่", "ณ", "ธ", "ก็"]: - return "karu" - else: - return "lahu" - - def check_klon(self, text: str, k_type: int = 8) -> list[str] | str: - """Check the suitability of the poem according to Thai principles. - - :param str text: Thai poem - :param int k_type: type of Thai poem - :return: the check results of the suitability of the poem according to Thai principles. - :rtype: Union[List[str], str] - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.check_klon( - 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง เอ๋งเอ๋งคะนอง \ - มีคนจับจอง เขาชื่อน้องเธียร', - k_type=4 - )) - # output: The poem is correct according to the principle. - - print(kv.check_klon( - 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง \ - เอ๋งเอ๋งเสียงหมา มีคนจับจอง เขาชื่อน้องเธียร', - k_type=4 - )) - # output: [ - "Can't find rhyme between paragraphs ('หมา', 'จอง') in paragraph 2", - "Can't find rhyme between paragraphs ('หมา', 'ทอง') in paragraph 2" - ] - """ - if k_type == 8: - try: - error = [] - list_sumpus_sent1 = [] - list_sumpus_sent2h = [] - list_sumpus_sent2l = [] - list_sumpus_sent3 = [] - list_sumpus_sent4 = [] - for i, sent in enumerate(text.split()): - sub_sent = subword_tokenize(sent, engine="dict") - if len(sub_sent) > 10: - error.append( - "In sentence " - + str(i + 2) - + ", there are more than 10 words. " - + str(sub_sent) - ) - if (i + 1) % 4 == 1: - list_sumpus_sent1.append(sub_sent[-1]) - elif (i + 1) % 4 == 2: - list_sumpus_sent2h.append( - [ - sub_sent[1], - sub_sent[2], - sub_sent[3], - sub_sent[4], - ] - ) - list_sumpus_sent2l.append(sub_sent[-1]) - elif (i + 1) % 4 == 3: - list_sumpus_sent3.append(sub_sent[-1]) - elif (i + 1) % 4 == 0: - list_sumpus_sent4.append(sub_sent[-1]) - if ( - len(list_sumpus_sent1) != len(list_sumpus_sent2h) - or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) - or len(list_sumpus_sent2l) != len(list_sumpus_sent3) - or len(list_sumpus_sent3) != len(list_sumpus_sent4) - or len(list_sumpus_sent4) != len(list_sumpus_sent1) - ): - return "The poem does not have 4 complete sentences." - else: - for i in range(len(list_sumpus_sent1)): - countwrong = 0 - for j in list_sumpus_sent2h[i]: - if ( - self.is_sumpus(list_sumpus_sent1[i], j) - is False - ): - countwrong += 1 - if countwrong > 3: - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent1[i], - list_sumpus_sent2h[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if ( - self.is_sumpus( - list_sumpus_sent2l[i], list_sumpus_sent3[i] - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent3[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if i > 0: - if ( - self.is_sumpus( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if not error: - return ( - "The poem is correct according to the principle." - ) - else: - return error - except: - return ("Something went wrong. " - "Make sure you enter it in the correct form of klon 8.") - elif k_type == 4: - try: - error = [] - list_sumpus_sent1 = [] - list_sumpus_sent2h = [] - list_sumpus_sent2l = [] - list_sumpus_sent3 = [] - list_sumpus_sent4 = [] - for i, sent in enumerate(text.split()): - sub_sent = subword_tokenize(sent, engine="dict") - if len(sub_sent) > 5: - error.append( - "In sentence " - + str(i + 2) - + ", there are more than 4 words. " - + str(sub_sent) - ) - if (i + 1) % 4 == 1: - list_sumpus_sent1.append(sub_sent[-1]) - elif (i + 1) % 4 == 2: - list_sumpus_sent2h.append([sub_sent[1], sub_sent[2]]) - list_sumpus_sent2l.append(sub_sent[-1]) - elif (i + 1) % 4 == 3: - list_sumpus_sent3.append(sub_sent[-1]) - elif (i + 1) % 4 == 0: - list_sumpus_sent4.append(sub_sent[-1]) - if ( - len(list_sumpus_sent1) != len(list_sumpus_sent2h) - or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) - or len(list_sumpus_sent2l) != len(list_sumpus_sent3) - or len(list_sumpus_sent3) != len(list_sumpus_sent4) - or len(list_sumpus_sent4) != len(list_sumpus_sent1) - ): - return "The poem does not have 4 complete sentences." - else: - for i in range(len(list_sumpus_sent1)): - countwrong = 0 - for j in list_sumpus_sent2h[i]: - if ( - self.is_sumpus(list_sumpus_sent1[i], j) - is False - ): - countwrong += 1 - if countwrong > 1: - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent1[i], - list_sumpus_sent2h[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if ( - self.is_sumpus( - list_sumpus_sent2l[i], list_sumpus_sent3[i] - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent3[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if i > 0: - if ( - self.is_sumpus( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if not error: - return ( - "The poem is correct according to the principle." - ) - else: - return error - except: - return "Something went wrong. Make sure you enter it in the correct form." - - else: - return "Something went wrong. Make sure you enter it in the correct form." - - def check_aek_too( - self, text: list[str] | str, dead_syllable_as_aek: bool = False - ) -> list[bool] | list[str] | bool | str: - """Checker of Thai tonal words - - :param Union[List[str], str] text: Thai word or list of Thai words - :param bool dead_syllable_as_aek: if True, dead syllable will be - considered as aek - :return: the check result if the word is aek or too or False (not both) - or list of check results if input is list - :rtype: Union[List[bool], List[str], bool, str] - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - # การเช็คคำเอกโท - print( - kv.check_aek_too("เอง"), - kv.check_aek_too("เอ่ง"), - kv.check_aek_too("เอ้ง"), - ) - # -> False, aek, too - print(kv.check_aek_too(["เอง", "เอ่ง", "เอ้ง"])) # ใช้ List ได้เหมือนกัน - # -> [False, 'aek', 'too'] - - - """ - if isinstance(text, list): - return [self.check_aek_too(t, dead_syllable_as_aek) for t in text] - - if not isinstance(text, str): - raise TypeError("text must be str or iterable list[str]") - - word_characters = [*text] - if "่" in word_characters and "้" not in word_characters: - return "aek" - elif "้" in word_characters and "่" not in word_characters: - return "too" - if dead_syllable_as_aek and sound_syllable(text) == "dead": - return "aek" - else: - return False - - def handle_karun_sound_silence(self, word: str) -> str: - """Handle silent sounds in Thai words using '์' character (Karun) - by stripping all characters before the 'Karun' character that should be silenced - - :param str text: Thai word - :return: Thai word with silent words stripped - :rtype: str - """ - sound_silenced = word.endswith("์") - if not sound_silenced: - return word - # Remove ์ and the silent consonant before it - # การันต์ (์) marks the consonant immediately before it as silent - word = word[:-2] - return word +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: C901 +from __future__ import annotations + +from pythainlp import thai_consonants +from pythainlp.tokenize import subword_tokenize +from pythainlp.util import remove_tonemark, sound_syllable + + +class KhaveeVerifier: + def __init__(self): + """ + KhaveeVerifier: Thai Poetry verifier + """ + + def _has_true_final_yl(self, word: str) -> bool: + """ + Check if ย or ล is a true final consonant + (not just part of the vowel sound with ไ/ใ) + + :param str word: Thai word + :return: True if ย or ล is a true final consonant + :rtype: bool + """ + if len(word) < 2: + return False + # Count consonants in the word + consonant_count = sum(1 for c in word if c in thai_consonants) + # If there are 2+ consonants and word ends with ย or ล, it's a true final + return consonant_count >= 2 and word[-1] in ["ย", "ล"] + + def check_sara(self, word: str) -> str: + """ + Check the vowels in the Thai word. + + :param str word: Thai word + :return: vowel name of the word + :rtype: str + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.check_sara("เริง")) + # output: 'เออ' + """ + sara = [] + countoa = 0 + + # In case of การันย์ + if "์" in word[-1]: + word = word[:-2] + + # In case of สระเดี่ยว + for i in word: + if i in ("ะ", "ั"): + sara.append("อะ") + elif i == "ิ": + sara.append("อิ") + elif i == "ุ": + sara.append("อุ") + elif i == "ึ": + sara.append("อึ") + elif i == "ี": + sara.append("อี") + elif i == "ู": + sara.append("อู") + elif i == "ื": + sara.append("อือ") + elif i == "เ": + sara.append("เอ") + elif i == "แ": + sara.append("แอ") + elif i == "า": + sara.append("อา") + elif i == "โ": + sara.append("โอ") + elif i == "ำ": + sara.append("อำ") + elif i == "อ": + countoa += 1 + sara.append("ออ") + elif i == "ั" and "ว" in word: + sara.append("อัว") + elif i in ("ไ", "ใ"): + sara.append("ไอ") + elif i == "็": + sara.append("ออ") + elif "รร" in word: + if self.check_marttra(word) == "กม": + sara.append("อำ") + else: + sara.append("อะ") + + # In case of ออ + if countoa == 1 and "อ" in word[-1] and "เ" not in word: + sara.remove("ออ") + + # In case of เอ เอ + countA = 0 + for i in sara: + if i == "เอ": + countA = countA + 1 + if countA > 1: + sara.remove("เอ") + sara.remove("เอ") + sara.append("แ") + + # In case of สระประสม + if "เอ" in sara and "อะ" in sara: + sara.remove("เอ") + sara.remove("อะ") + sara.append("เอะ") + elif "แอ" in sara and "อะ" in sara: + sara.remove("แอ") + sara.remove("อะ") + sara.append("แอะ") + + if "เอะ" in sara and "ออ" in sara: + sara.remove("เอะ") + sara.remove("ออ") + sara.append("เออะ") + elif "เอ" in sara and "อิ" in sara: + sara.remove("เอ") + sara.remove("อิ") + sara.append("เออ") + elif "เอ" in sara and "ออ" in sara and "อ" in word[-1]: + sara.remove("เอ") + sara.remove("ออ") + sara.append("เออ") + elif "โอ" in sara and "อะ" in sara: + sara.remove("โอ") + sara.remove("อะ") + sara.append("โอะ") + elif "เอ" in sara and "อี" in sara: + sara.remove("เอ") + sara.remove("อี") + sara.append("เอีย") + elif "เอ" in sara and "อือ" in sara: + sara.remove("เอ") + sara.remove("อือ") + sara.append("อัว") + elif "เอ" in sara and "อา" in sara: + sara.remove("เอ") + sara.remove("อา") + sara.append("เอา") + elif "เ" in word and "า" in word and "ะ" in word: + sara = [] + sara.append("เอาะ") + + if "อือ" in sara and "เออ" in sara: + sara.remove("เออ") + sara.remove("อือ") + sara.append("เอือ") + elif "ออ" in sara and len(sara) > 1: + sara.remove("ออ") + elif "ว" in word and len(sara) == 0: + sara.append("อัว") + + if "ั" in word and self.check_marttra(word) == "กา": + sara = [] + sara.append("ไอ") + + # In case of อ + if word == "เออะ": + sara = [] + sara.append("เออะ") + elif word == "เออ": + sara = [] + sara.append("เออ") + elif word == "เอ": + sara = [] + sara.append("เอ") + elif word == "เอะ": + sara = [] + sara.append("เอะ") + elif word == "เอา": + sara = [] + sara.append("เอา") + elif word == "เอาะ": + sara = [] + sara.append("เอาะ") + + if "ฤา" in word or "ฦา" in word: + sara = [] + sara.append("อือ") + elif "ฤ" in word or "ฦ" in word: + sara = [] + sara.append("อึ") + + # In case of กน + if not sara and len(word) == 2: + if word[-1] != "ร": + sara.append("โอะ") + else: + sara.append("ออ") + elif not sara and len(word) == 3: + sara.append("ออ") + + # In case of บ่ + if word == "บ่": + sara = [] + sara.append("ออ") + + if "ํ" in word: + sara = [] + sara.append("อำ") + + if "เ" in word and "ื" in word and "อ" in word: + sara = [] + sara.append("เอือ") + + if not sara: + return "Can't find Sara in this word" + + return sara[0] + + def check_marttra(self, word: str) -> str: + """ + Check the Thai spelling Section in the Thai word. + + :param str word: Thai word + :return: name of spelling Section of the word. + :rtype: str + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.check_marttra("สาว")) + # output: 'เกอว' + """ + # Handle consonant clusters ending with ร + # ตร, ทร → remove ร (treat as final ต/ท sound) + # กร, ขร, คร, ฆร in compound words → remove ร (treat as final ก/ข/ค sound) + # But single syllable words like "กร" should keep ร + if len(word) >= 3 and word[-1] == "ร": + if word[-2] in ["ต", "ท"]: + word = word[:-1] + elif word[-2] in ["ก", "ข", "ค", "ฆ"]: + word = word[:-1] + + word = self.handle_karun_sound_silence(word) + word = remove_tonemark(word) + + # Check for ำ at the end (represents "am" sound, ends with m) + if word[-1] == "ำ": + return "กม" + + # Check for vowels and special patterns that indicate open syllables (กา) + # For words with ไ/ใ, check if ย/ล is a true final or just part of vowel + if "ไ" in word or "ใ" in word: + if word[-1] not in ["ย", "ล"]: + return "กา" + elif not self._has_true_final_yl(word): + # ย/ล is part of the vowel sound, not a true final + return "กา" + # else: ย/ล is a true final, continue to consonant classification below + + if "ํ" in word and "า" in word: + return "กา" + elif ( + word[-1] in ["า", "ะ", "ิ", "ี", "ุ", "ู", "อ"] + or ("ี" in word and "ย" in word[-1]) + or ("ื" in word and "อ" in word[-1]) + ): + return "กา" + elif word[-1] in ["ง"]: + return "กง" + elif word[-1] in ["ม"]: + return "กม" + elif word[-1] in ["ย"]: + return "เกย" + elif word[-1] in ["ล"]: + return "เกย" + elif word[-1] in ["ว"]: + return "เกอว" + elif word[-1] in ["ก", "ข", "ค", "ฆ"]: + return "กก" + elif word[-1] in [ + "จ", + "ช", + "ซ", + "ฎ", + "ฏ", + "ฐ", + "ฑ", + "ฒ", + "ด", + "ต", + "ถ", + "ท", + "ธ", + "ศ", + "ษ", + "ส", + ]: + return "กด" + elif word[-1] in ["ญ", "ณ", "น", "ร", "ฬ"]: + return "กน" + elif word[-1] in ["บ", "ป", "พ", "ฟ", "ภ"]: + return "กบ" + else: + if "็" in word: + return "กา" + else: + return "Cant find Marttra in this word" + + def is_sumpus(self, word1: str, word2: str) -> bool: + """ + Check the rhyme between two words. + + :param str word1: Thai word + :param str word2: Thai word + :return: boolean + :rtype: bool + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.is_sumpus("สรร", "อัน")) + # output: True + + print(kv.is_sumpus("สรร", "แมว")) + # output: False + """ + marttra1 = self.check_marttra(word1) + marttra2 = self.check_marttra(word2) + sara1 = self.check_sara(word1) + sara2 = self.check_sara(word2) + if sara1 == "อะ" and marttra1 == "เกย": + sara1 = "ไอ" + marttra1 = "กา" + elif sara2 == "อะ" and marttra2 == "เกย": + sara2 = "ไอ" + marttra2 = "กา" + if sara1 == "อำ" and marttra1 == "กม": + sara1 = "อำ" + marttra1 = "กา" + elif sara2 == "อำ" and marttra2 == "กม": + sara2 = "อำ" + marttra2 = "กา" + return bool(marttra1 == marttra2 and sara1 == sara2) + + def check_karu_lahu(self, text): + if ( + self.check_marttra(text) != "กา" + or ( + self.check_marttra(text) == "กา" + and self.check_sara(text) + in [ + "อา", + "อี", + "อือ", + "อู", + "เอ", + "แอ", + "โอ", + "ออ", + "เออ", + "เอีย", + "เอือ", + "อัว", + ] + ) + or self.check_sara(text) in ["อำ", "ไอ", "เอา"] + ) and text not in ["บ่", "ณ", "ธ", "ก็"]: + return "karu" + else: + return "lahu" + + def check_klon(self, text: str, k_type: int = 8) -> list[str] | str: + """ + Check the suitability of the poem according to Thai principles. + + :param str text: Thai poem + :param int k_type: type of Thai poem + :return: the check results of the suitability of the + poem according to Thai principles. + :rtype: Union[List[str], str] + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.check_klon( + 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง เอ๋งเอ๋งคะนอง \ + มีคนจับจอง เขาชื่อน้องเธียร', + k_type=4 + )) + # output: The poem is correct according to the principle. + + print(kv.check_klon( + 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง \ + เอ๋งเอ๋งเสียงหมา มีคนจับจอง เขาชื่อน้องเธียร', + k_type=4 + )) + # output: [ + "Can't find rhyme between paragraphs ('หมา', 'จอง') in paragraph 2", + "Can't find rhyme between paragraphs ('หมา', 'ทอง') in paragraph 2" + ] + """ + if k_type == 8: + try: + error = [] + list_sumpus_sent1 = [] + list_sumpus_sent2h = [] + list_sumpus_sent2l = [] + list_sumpus_sent3 = [] + list_sumpus_sent4 = [] + for i, sent in enumerate(text.split()): + sub_sent = subword_tokenize(sent, engine="dict") + if len(sub_sent) > 10: + error.append( + "In sentence " + + str(i + 2) + + ", there are more than 10 words. " + + str(sub_sent) + ) + if (i + 1) % 4 == 1: + list_sumpus_sent1.append(sub_sent[-1]) + elif (i + 1) % 4 == 2: + list_sumpus_sent2h.append( + [ + sub_sent[1], + sub_sent[2], + sub_sent[3], + sub_sent[4], + ] + ) + list_sumpus_sent2l.append(sub_sent[-1]) + elif (i + 1) % 4 == 3: + list_sumpus_sent3.append(sub_sent[-1]) + elif (i + 1) % 4 == 0: + list_sumpus_sent4.append(sub_sent[-1]) + if ( + len(list_sumpus_sent1) != len(list_sumpus_sent2h) + or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) + or len(list_sumpus_sent2l) != len(list_sumpus_sent3) + or len(list_sumpus_sent3) != len(list_sumpus_sent4) + or len(list_sumpus_sent4) != len(list_sumpus_sent1) + ): + return "The poem does not have 4 complete sentences." + else: + for i in range(len(list_sumpus_sent1)): + countwrong = 0 + for j in list_sumpus_sent2h[i]: + if ( + self.is_sumpus(list_sumpus_sent1[i], j) + is False + ): + countwrong += 1 + if countwrong > 3: + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent1[i], + list_sumpus_sent2h[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if ( + self.is_sumpus( + list_sumpus_sent2l[i], list_sumpus_sent3[i] + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent3[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if i > 0: + if ( + self.is_sumpus( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if not error: + return ( + "The poem is correct according to the principle." + ) + else: + return error + except: + return "Something went wrong. Make sure you enter it in the correct form of klon 8." + elif k_type == 4: + try: + error = [] + list_sumpus_sent1 = [] + list_sumpus_sent2h = [] + list_sumpus_sent2l = [] + list_sumpus_sent3 = [] + list_sumpus_sent4 = [] + for i, sent in enumerate(text.split()): + sub_sent = subword_tokenize(sent, engine="dict") + if len(sub_sent) > 5: + error.append( + "In sentence " + + str(i + 2) + + ", there are more than 4 words. " + + str(sub_sent) + ) + if (i + 1) % 4 == 1: + list_sumpus_sent1.append(sub_sent[-1]) + elif (i + 1) % 4 == 2: + list_sumpus_sent2h.append([sub_sent[1], sub_sent[2]]) + list_sumpus_sent2l.append(sub_sent[-1]) + elif (i + 1) % 4 == 3: + list_sumpus_sent3.append(sub_sent[-1]) + elif (i + 1) % 4 == 0: + list_sumpus_sent4.append(sub_sent[-1]) + if ( + len(list_sumpus_sent1) != len(list_sumpus_sent2h) + or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) + or len(list_sumpus_sent2l) != len(list_sumpus_sent3) + or len(list_sumpus_sent3) != len(list_sumpus_sent4) + or len(list_sumpus_sent4) != len(list_sumpus_sent1) + ): + return "The poem does not have 4 complete sentences." + else: + for i in range(len(list_sumpus_sent1)): + countwrong = 0 + for j in list_sumpus_sent2h[i]: + if ( + self.is_sumpus(list_sumpus_sent1[i], j) + is False + ): + countwrong += 1 + if countwrong > 1: + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent1[i], + list_sumpus_sent2h[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if ( + self.is_sumpus( + list_sumpus_sent2l[i], list_sumpus_sent3[i] + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent3[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if i > 0: + if ( + self.is_sumpus( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if not error: + return ( + "The poem is correct according to the principle." + ) + else: + return error + except: + return "Something went wrong. Make sure you enter it in the correct form." + + else: + return "Something went wrong. Make sure you enter it in the correct form." + + def check_aek_too( + self, text: list[str] | str, dead_syllable_as_aek: bool = False + ) -> list[bool] | list[str] | bool | str: + """ + Checker of Thai tonal words + + :param Union[List[str], str] text: Thai word or list of Thai words + :param bool dead_syllable_as_aek: if True, dead syllable will + be considered as aek + :return: the check result if the word is aek or too + or False (not both) or list of check results if input is list + :rtype: Union[List[bool], List[str], bool, str] + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + # การเช็คคำเอกโท + print( + kv.check_aek_too("เอง"), + kv.check_aek_too("เอ่ง"), + kv.check_aek_too("เอ้ง"), + ) + # -> False, aek, too + print(kv.check_aek_too(["เอง", "เอ่ง", "เอ้ง"])) # ใช้ List ได้เหมือนกัน + # -> [False, 'aek', 'too'] + """ + if isinstance(text, list): + return [self.check_aek_too(t, dead_syllable_as_aek) for t in text] + + if not isinstance(text, str): + raise TypeError("text must be str or iterable list[str]") + + word_characters = [*text] + if "่" in word_characters and "้" not in word_characters: + return "aek" + elif "้" in word_characters and "่" not in word_characters: + return "too" + if dead_syllable_as_aek and sound_syllable(text) == "dead": + return "aek" + else: + return False + + def handle_karun_sound_silence(self, word: str) -> str: + """ + Handle silent sounds in Thai words using '์' character (Karun) + by stripping all characters before the 'Karun' character + that should be silenced + + :param str text: Thai word + :return: Thai word with silent words stripped + :rtype: str + """ + sound_silenced = word.endswith("์") + if not sound_silenced: + return word + # Remove ์ and the silent consonant before it + # การันต์ (์) marks the consonant immediately before it as silent + word = word[:-2] + return word diff --git a/pythainlp/lm/text_util.py b/pythainlp/lm/text_util.py index 26fd4fa94..f2f73cce2 100644 --- a/pythainlp/lm/text_util.py +++ b/pythainlp/lm/text_util.py @@ -17,6 +17,9 @@ def calculate_ngram_counts( :return: A dictionary where keys are n-grams and values are their counts. :rtype: Dict[Tuple[str], int] """ + if not list_words: + return {} + ngram_counts = {} for n in range(n_min, n_max + 1): diff --git a/pythainlp/morpheme/word_formation.py b/pythainlp/morpheme/word_formation.py index 212c3e23a..63453a25e 100644 --- a/pythainlp/morpheme/word_formation.py +++ b/pythainlp/morpheme/word_formation.py @@ -35,7 +35,7 @@ def nighit(w1: str, w2: str) -> str: raise NotImplementedError(f"The function doesn't support {w1}.") list_w1 = list(w1) list_w2 = list(w2) - newword = list() + newword = [] newword.append(list_w1[0]) newword.append("ั") consonant_start = [i for i in list_w2 if i in set(thai_consonants)][0] diff --git a/pythainlp/spell/pn.py b/pythainlp/spell/pn.py index 695bc7cf1..361a000c2 100644 --- a/pythainlp/spell/pn.py +++ b/pythainlp/spell/pn.py @@ -10,6 +10,7 @@ from collections import Counter from collections.abc import Callable, ItemsView, Iterable from string import digits +from typing import cast from pythainlp import thai_digits, thai_letters from pythainlp.corpus import tnc @@ -34,7 +35,7 @@ def _keep( min_freq: int, min_len: int, max_len: int, - dict_filter: Callable[[str], bool], + dict_filter: Callable[[str], bool] | None, ) -> bool: """Checks whether a given word has the required minimum frequency min_freq and its character length is between min_len and max_len (inclusive). @@ -46,6 +47,9 @@ def _keep( if not (word and min_len <= len(word) <= max_len and word[0] != "."): return False + if not dict_filter: + dict_filter = _no_filter + return dict_filter(word) @@ -58,7 +62,13 @@ def _edits1(word: str) -> set[str]: replaces = [L + c + R[1:] for L, R in splits if R for c in thai_letters] inserts = [L + c + R for L, R in splits for c in thai_letters] - return set(deletes + transposes + replaces + inserts) + # Use set union for better performance than list concatenation + result = set(deletes) + result.update(transposes) + result.update(replaces) + result.update(inserts) + + return result def _edits2(word: str) -> set[str]: @@ -81,17 +91,22 @@ def _convert_custom_dict( i = iter(custom_dict) first_member = next(i) + + result: list[tuple[str, int]] + if isinstance(first_member, str): # create tuples of a word with frequency equaling 1, # and filter word list - custom_dict = [ + custom_dict = cast(Iterable[str], custom_dict) + result = [ (word, 1) for word in custom_dict if _keep((word, 1), 1, min_len, max_len, dict_filter) ] elif isinstance(first_member, tuple): # filter word list - custom_dict = [ + custom_dict = cast(Iterable[tuple[str, int]], custom_dict) + result = [ word_freq for word_freq in custom_dict if _keep(word_freq, min_freq, min_len, max_len, dict_filter) @@ -102,15 +117,15 @@ def _convert_custom_dict( "Iterable[Tuple[str, int]], or Iterable[str]" ) - return custom_dict + return result class NorvigSpellChecker: def __init__( self, - custom_dict: dict[str, int] - | Iterable[str] - | Iterable[tuple[str, int]] = None, + custom_dict: ( + dict[str, int] | Iterable[str] | Iterable[tuple[str, int]] | None + ) = None, min_freq: int = 2, min_len: int = 2, max_len: int = 40, diff --git a/pythainlp/tag/blackboard.py b/pythainlp/tag/blackboard.py index 2a3938f14..42c43e2c3 100644 --- a/pythainlp/tag/blackboard.py +++ b/pythainlp/tag/blackboard.py @@ -5,7 +5,7 @@ # defined strings for special characters CHAR_TO_ESCAPE = {" ": "_"} -ESCAPE_TO_CHAR = dict((v, k) for k, v in CHAR_TO_ESCAPE.items()) +ESCAPE_TO_CHAR = {v: k for k, v in CHAR_TO_ESCAPE.items()} # map from Blackboard treebank POS tag to Universal POS tag diff --git a/pythainlp/tag/orchid.py b/pythainlp/tag/orchid.py index 69fcb0c07..f9a905a57 100644 --- a/pythainlp/tag/orchid.py +++ b/pythainlp/tag/orchid.py @@ -33,7 +33,7 @@ ";": "", "/": "", } -ESCAPE_TO_CHAR = dict((v, k) for k, v in CHAR_TO_ESCAPE.items()) +ESCAPE_TO_CHAR = {v: k for k, v in CHAR_TO_ESCAPE.items()} # map from ORCHID POS tag to Universal POS tag # from Korakot Chaovavanich diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index 9259041dc..f7b6d9d10 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -6,8 +6,8 @@ from __future__ import annotations -import copy import re +from collections import deque from collections.abc import Iterable from pythainlp.tokenize import ( @@ -25,6 +25,10 @@ ) from pythainlp.util.trie import Trie, dict_trie +_RE_WHITESPACE = re.compile(r"\s") +_RE_WORD_CHAR = re.compile(r"\w") + + def word_detokenize( segments: list[list[str]] | list[str], output: str = "str" @@ -337,24 +341,22 @@ def indices_words(words): def map_indices_to_words(index_list, sentences): result = [] - c = copy.copy(index_list) + c = deque(index_list) n_sum = 0 for sentence in sentences: words = sentence sentence_result = [] - n = 0 - for start, end in c: + while c: + start, end = c[0] if start > n_sum + len(words) - 1: break else: + c.popleft() word = sentence[start - n_sum : end + 1 - n_sum] sentence_result.append(word) - n += 1 result.append(sentence_result) n_sum += len(words) - for _ in range(n): - del c[0] return result @@ -459,7 +461,7 @@ def sent_tokenize( result = [] _temp: list[str] = [] for i, w in enumerate(text): - if re.findall(r" ", w) != [] and re.findall(r"\w", w) == []: + if " " in w and not _RE_WORD_CHAR.search(w): if not _temp: continue result.append(_temp) @@ -475,9 +477,7 @@ def sent_tokenize( result = [] _temp = [] for i, w in enumerate(text): - if ( - re.findall(r"\s", w) != [] or re.findall(r"\n", w) != [] - ) and re.findall(r"\w", w) == []: + if _RE_WHITESPACE.search(w) and not _RE_WORD_CHAR.search(w): if not _temp: continue result.append(_temp) diff --git a/pythainlp/tokenize/crfcut.py b/pythainlp/tokenize/crfcut.py index b4cb9e0be..bbf437b05 100644 --- a/pythainlp/tokenize/crfcut.py +++ b/pythainlp/tokenize/crfcut.py @@ -138,26 +138,19 @@ def extract_features( within the `window` :return: list of lists of features to be fed to CRF """ + if not doc: + return [] + doc_features = [] - doc = ( - ["xxpad" for i in range(window)] - + doc - + ["xxpad" for i in range(window)] - ) + # Pad the document with "xxpad" tokens efficiently + padded_doc = ["xxpad"] * window + padded_doc.extend(doc) + padded_doc.extend(["xxpad"] * window) + doc = padded_doc # add enders and starters - doc_ender = [] - doc_starter = [] - for i in range(len(doc)): - if doc[i] in _ENDERS: - doc_ender.append("ender") - else: - doc_ender.append("normal") - - if doc[i] in _STARTERS: - doc_starter.append("starter") - else: - doc_starter.append("normal") + doc_ender = ["ender" if token in _ENDERS else "normal" for token in doc] + doc_starter = ["starter" if token in _STARTERS else "normal" for token in doc] # for each word for i in range(window, len(doc) - window): diff --git a/pythainlp/tokenize/thaisumcut.py b/pythainlp/tokenize/thaisumcut.py index 35dda6ec0..9909c0c30 100644 --- a/pythainlp/tokenize/thaisumcut.py +++ b/pythainlp/tokenize/thaisumcut.py @@ -29,18 +29,24 @@ def list_to_string(list: list[str]) -> str: def middle_cut(sentences: list[str]) -> list[str]: - new_text = "" + if not sentences: + return [] + + result_parts = [] for sentence in sentences: sentence_size = len(word_tokenize(sentence, keep_whitespace=False)) - for k in range(0, len(sentence)): - if k == 0 or k + 1 >= len(sentence): + sentence_len = len(sentence) + for k in range(0, sentence_len): + if k == 0 or k + 1 >= sentence_len: continue if sentence[k].isdigit() and sentence[k - 1] == " ": sentence = sentence[: k - 1] + sentence[k:] - if k + 2 <= len(sentence): + sentence_len = len(sentence) # Update length after modification + if k + 2 <= sentence_len: if sentence[k].isdigit() and sentence[k + 1] == " ": sentence = sentence[: k + 1] + sentence[k + 2 :] + sentence_len = len(sentence) # Update length after modification fixed_text_lenth = 20 @@ -52,44 +58,35 @@ def middle_cut(sentences: list[str]) -> list[str]: white_space_index = [] white_space_diff = {} - for j in range(len(tokens)): - if tokens[j] == " ": + for j, token in enumerate(tokens): + if token == " ": white_space_index.append(j) for white_space in white_space_index: - white_space_diff.update( - {white_space: abs(white_space - middle_space)} - ) - - if len(white_space_diff) > 0: - min_diff = min( - white_space_diff.items(), key=operator.itemgetter(1) - ) + white_space_diff[white_space] = abs(white_space - middle_space) + + if white_space_diff: + min_diff = min(white_space_diff.items(), key=operator.itemgetter(1)) tokens.pop(min_diff[0]) tokens.insert(min_diff[0], "") - new_text = new_text + list_to_string(tokens) + "" + result_parts.append(list_to_string(tokens)) else: - new_text = new_text + sentence + "" + result_parts.append(sentence) - sentences = new_text.split("") - sentences = [s.strip() for s in sentences] - if "" in sentences: - sentences.remove("") - if "nan" in sentences: - sentences.remove("nan") + # Split all result parts by and filter + all_sentences = (s.strip() for part in result_parts for s in part.split("")) - sentences = list(filter(None, sentences)) - return sentences + return [s for s in all_sentences if s] class ThaiSentenceSegmentor: - def split_into_sentences( - self, text: str, isMiddleCut: bool = False - ) -> list[str]: + def split_into_sentences(self, text: str, isMiddleCut: bool = False) -> list[str]: # Declare Variables th_alphabets = "([ก-๙])" th_conjunction = "(ทำให้|โดย|เพราะ|นอกจากนี้|แต่|กรณีที่|หลังจากนี้|ต่อมา|ภายหลัง|นับตั้งแต่|หลังจาก|ซึ่งเหตุการณ์|ผู้สื่อข่าวรายงานอีก|ส่วนที่|ส่วนสาเหตุ|ฉะนั้น|เพราะฉะนั้น|เพื่อ|เนื่องจาก|จากการสอบสวนทราบว่า|จากกรณี|จากนี้|อย่างไรก็ดี)" - th_cite = "(กล่าวว่า|เปิดเผยว่า|รายงานว่า|ให้การว่า|เผยว่า|บนทวิตเตอร์ว่า|แจ้งว่า|พลเมืองดีว่า|อ้างว่า)" + th_cite = ( + "(กล่าวว่า|เปิดเผยว่า|รายงานว่า|ให้การว่า|เผยว่า|บนทวิตเตอร์ว่า|แจ้งว่า|พลเมืองดีว่า|อ้างว่า)" + ) th_ka_krub = "(ครับ|ค่ะ)" th_stop_after = "(หรือไม่|โดยเร็ว|แล้ว|อีกด้วย)" th_stop_before = "(ล่าสุด|เบื้องต้น|ซึ่ง|ทั้งนี้|แม้ว่า|เมื่อ|แถมยัง|ตอนนั้น|จนเป็นเหตุให้|จากนั้น|อย่างไรก็ตาม|และก็|อย่างใดก็ตาม|เวลานี้|เช่น|กระทั่ง)" @@ -155,9 +152,7 @@ def split_into_sentences( text = text.replace("ทั้งนี้เพื่อ", "ทั้งนี้") text = text.replace("เวลาต่อมา", "เวลา") text = text.replace("อย่างไรก็ตาม", "อย่างไรก็ตาม") - text = text.replace( - "อย่างไรก็ตามหลังจาก", "อย่างไรก็ตาม" - ) + text = text.replace("อย่างไรก็ตามหลังจาก", "อย่างไรก็ตาม") text = text.replace("ซึ่งทำให้", "ซึ่ง") text = text.replace("โดยประมาท", "ประมาท") text = text.replace("โดยธรรม", "ธรรม") @@ -170,14 +165,14 @@ def split_into_sentences( last_position = len(tokens) pop_split_position = [] split_position = [] - for i in range(len(tokens)): - if tokens[i] == "และ": + for i, token in enumerate(tokens): + if token == "และ": and_position = i if ( and_position != -1 and i > and_position - and tokens[i] == " " + and token == " " and nearest_space_position == -1 ): if i - and_position != 1: @@ -209,13 +204,13 @@ def split_into_sentences( last_position = len(tokens) pop_split_position = [] split_position = [] - for i in range(len(tokens)): - if tokens[i] == "หรือ": + for i, token in enumerate(tokens): + if token == "หรือ": or_position = i if ( or_position != -1 and i > or_position - and tokens[i] == " " + and token == " " and nearest_space_position == -1 ): if i - or_position != 1: @@ -247,13 +242,13 @@ def split_into_sentences( pop_split_position = [] last_position = len(tokens) split_position = [] - for i in range(len(tokens)): - if tokens[i] == "จึง": + for i, token in enumerate(tokens): + if token == "จึง": cung_position = i if ( cung_position != -1 - and tokens[i] == " " + and token == " " and i > cung_position and nearest_space_position == -1 ): @@ -286,9 +281,7 @@ def split_into_sentences( text = re.sub(th_conjunction, "\\1", text) text = re.sub(th_cite, "\\1", text) text = re.sub(" " + degit + "[.]" + th_title, "\\1.\\2", text) - text = re.sub( - " " + degit + degit + "[.]" + th_title, "\\1\\2.\\3", text - ) + text = re.sub(" " + degit + degit + "[.]" + th_title, "\\1\\2.\\3", text) text = re.sub(th_alphabets + th_stop_after + " ", "\\1\\2", text) if "”" in text: text = text.replace(".”", "”.") diff --git a/pythainlp/transliterate/thai2rom_onnx.py b/pythainlp/transliterate/thai2rom_onnx.py index 19ebb0468..ab00131d2 100644 --- a/pythainlp/transliterate/thai2rom_onnx.py +++ b/pythainlp/transliterate/thai2rom_onnx.py @@ -127,9 +127,9 @@ def run(self, source_seq, source_seq_len): outputs = np.zeros((max_len, batch_size, self.target_vocab_size)) - expected_encoder_outputs = list( - map(lambda output: output.name, self.encoder.get_outputs()) - ) + expected_encoder_outputs = [ + output.name for output in self.encoder.get_outputs() + ] encoder_outputs, encoder_hidden, _ = self.encoder.run( input_feed={ "input_tensor": source_seq, diff --git a/pythainlp/util/wordtonum.py b/pythainlp/util/wordtonum.py index c213b6ec1..f56dd88a4 100644 --- a/pythainlp/util/wordtonum.py +++ b/pythainlp/util/wordtonum.py @@ -55,7 +55,7 @@ def _tokenizer(): def _check_is_thainum(word: str): - for j in list(_digits.keys()): + for j in _digits: if j in word: return (True, "num") for j in ["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด", "ลบ"]: @@ -68,10 +68,10 @@ def _check_is_thainum(word: str): def _tokenizer_thaiwords(): """Lazy load Thai words tokenizer with cache""" _dict_words = [ - i for i in list(thai_words()) if not _check_is_thainum(i)[0] + i for i in thai_words() if not _check_is_thainum(i)[0] ] - _dict_words += list(_digits.keys()) - _dict_words += ["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"] + _dict_words.extend(_digits.keys()) + _dict_words.extend(["สิบ", "ร้อย", "พัน", "หมื่น", "แสน", "ล้าน", "จุด"]) return Tokenizer(_dict_words) diff --git a/pythainlp/wsd/core.py b/pythainlp/wsd/core.py index 04e5da796..669b74dcb 100644 --- a/pythainlp/wsd/core.py +++ b/pythainlp/wsd/core.py @@ -13,8 +13,8 @@ for i, j in zip(_wsd_dict["word"], _wsd_dict["meaning"]): _mean_all[i] = j -_all_word = set(list(_mean_all.keys())) -_TRIE = Trie(list(_all_word)) +_all_word = set(_mean_all.keys()) +_TRIE = Trie(_all_word) _word_cut = Tokenizer(custom_dict=_TRIE) _MODEL = None @@ -50,7 +50,7 @@ def get_sense( sentence: str, word: str, device: str = "cpu", - custom_dict: dict = dict(), + custom_dict: dict | None = None, custom_tokenizer: Tokenizer = _word_cut, ) -> list[tuple[str, float]]: """Get word sense from the sentence.