diff --git a/pythainlp/augment/wordnet.py b/pythainlp/augment/wordnet.py index 80eb3aefe..309ab66f0 100644 --- a/pythainlp/augment/wordnet.py +++ b/pythainlp/augment/wordnet.py @@ -133,7 +133,7 @@ def find_synonyms( :param Optional[str] pos: part-of-speech type. Default is None. :param str postag_corpus: name of POS tag corpus :return: list of synonyms - :rtype: List[str] + :rtype: list[str] """ self.synonyms = [] if pos is None: @@ -171,7 +171,7 @@ def augment( :param str postag_corpus: name of POS tag corpus :return: list of synonyms - :rtype: List[Tuple[str]] + :rtype: list[list[str]] :Example: :: diff --git a/pythainlp/benchmarks/word_tokenization.py b/pythainlp/benchmarks/word_tokenization.py index 647abca43..2309157bb 100644 --- a/pythainlp/benchmarks/word_tokenization.py +++ b/pythainlp/benchmarks/word_tokenization.py @@ -5,6 +5,7 @@ import re import sys +from typing import Any import numpy as np import pandas as pd @@ -40,7 +41,7 @@ def _f1(precision: float, recall: float) -> float: return 2 * precision * recall / (precision + recall) -def _flatten_result(my_dict: dict, sep: str = ":") -> dict: +def _flatten_result(my_dict: dict, sep: str = ":") -> dict[str, Any]: """Flatten two-dimension dictionary. Use keys in the first dimension as a prefix for keys in the second dimension. @@ -54,7 +55,7 @@ def _flatten_result(my_dict: dict, sep: str = ":") -> dict: :param str sep: separator between the two keys (default: ":") :return: a one-dimension dictionary with keys combined - :rtype: dict[str, Union[float, str]] + :rtype: dict[str, Any] """ return { f"{k1}{sep}{k2}": v @@ -129,7 +130,7 @@ def preprocessing(txt: str, remove_space: bool = True) -> str: return txt -def compute_stats(ref_sample: str, raw_sample: str) -> dict: +def compute_stats(ref_sample: str, raw_sample: str) -> dict[str, Any]: """Compute statistics for tokenization quality These statistics include: @@ -146,7 +147,7 @@ def compute_stats(ref_sample: str, raw_sample: str) -> dict: :param str samples: samples that we want to evaluate :return: metrics at character- and word-level and indicators of correctly tokenized words - :rtype: dict[str, Union[float, str]] + :rtype: dict[str, Any] """ ref_sample_arr = _binary_representation(ref_sample) sample_arr = _binary_representation(raw_sample) @@ -222,7 +223,11 @@ def _binary_representation(txt: str, verbose: bool = False) -> np.ndarray: sample_wo_seps = list(txt.replace(SEPARATOR, "")) # sanity check - assert len(sample_wo_seps) == len(bin_rept) + if len(sample_wo_seps) != len(bin_rept): + raise ValueError( + f"Length mismatch: sample_wo_seps={len(sample_wo_seps)}, " + f"bin_rept={len(bin_rept)}" + ) if verbose: for c, m in zip(sample_wo_seps, bin_rept): @@ -231,13 +236,13 @@ def _binary_representation(txt: str, verbose: bool = False) -> np.ndarray: return bin_rept -def _find_word_boundaries(bin_reps) -> list: +def _find_word_boundaries(bin_reps) -> list[tuple[int, int]]: """Find the starting and ending location of each word. :param str bin_reps: binary representation of a text :return: list of tuples (start, end) - :rtype: list[tuple(int, int)] + :rtype: list[tuple[int, int]] """ boundary = np.argwhere(bin_reps == 1).reshape(-1) start_idx = boundary @@ -252,8 +257,8 @@ def _find_words_correctly_tokenised( ) -> tuple[int, ...]: """Find whether each word is correctly tokenized. - :param list[tuple(int, int)] ref_boundaries: word boundaries of reference tokenization - :param list[tuple(int, int)] predicted_boundaries: word boundareies of predicted tokenization + :param list[tuple[int, int]] ref_boundaries: word boundaries of reference tokenization + :param list[tuple[int, int]] predicted_boundaries: word boundaries of predicted tokenization :return: binary sequence where 1 indicates the corresponding word is tokenized correctly :rtype: tuple[int, ...] diff --git a/pythainlp/cli/benchmark.py b/pythainlp/cli/benchmark.py index b6da86d82..595551e1d 100644 --- a/pythainlp/cli/benchmark.py +++ b/pythainlp/cli/benchmark.py @@ -81,9 +81,10 @@ def __init__(self, name: str, argv: Sequence[str]) -> None: actual = _read_file(args.input_file) expected = _read_file(args.test_file) - assert len(actual) == len(expected), ( - "Input and test files do not have the same number of samples" - ) + if len(actual) != len(expected): + raise ValueError( + "Input and test files do not have the same number of samples" + ) safe_print( "Benchmarking %s against %s with %d samples in total" diff --git a/pythainlp/coref/core.py b/pythainlp/coref/core.py index 24da7554c..46e7aac1a 100644 --- a/pythainlp/coref/core.py +++ b/pythainlp/coref/core.py @@ -15,12 +15,12 @@ def coreference_resolution( ) -> list[dict]: """Coreference Resolution - :param List[str] texts: list of texts to apply coreference resolution to + :param Union[str, list[str]] texts: list of texts to apply coreference resolution to :param str model_name: coreference resolution model :param str device: device for running coreference resolution model on\ ("cpu", "cuda", and others) :return: List of texts with coreference resolution - :rtype: List[dict] + :rtype: list[dict] :Options for model_name: * *han-coref-v1.0* - (default) Han-Coref: Thai coreference resolution\ diff --git a/pythainlp/el/core.py b/pythainlp/el/core.py index 9af8cdf7d..e2c254147 100644 --- a/pythainlp/el/core.py +++ b/pythainlp/el/core.py @@ -42,9 +42,9 @@ def get_el( ) -> Union[list[dict], str]: """Get Entity Linking from Thai Text - :param str Union[List[str], str]: list of Thai text or text + :param str Union[list[str], str]: list of Thai text or text :return: list of entity linking - :rtype: Union[List[dict], str] + :rtype: Union[list[dict], str] :Example: :: diff --git a/pythainlp/generate/core.py b/pythainlp/generate/core.py index 34d9fa636..86e2cdb21 100644 --- a/pythainlp/generate/core.py +++ b/pythainlp/generate/core.py @@ -58,7 +58,7 @@ def gen_sentence( :param bool duplicate: allow duplicate words in sentence :return: list of words or a word string - :rtype: List[str], str + :rtype: list[str], str :Example: :: @@ -153,7 +153,7 @@ def gen_sentence( :param bool duplicate: allow duplicate words in sentence :return: list of words or a word string - :rtype: List[str], str + :rtype: list[str], str :Example: :: @@ -244,7 +244,7 @@ def gen_sentence( :param bool duplicate: allow duplicate words in sentence :return: list of words or a word string - :rtype: List[str], str + :rtype: list[str], str :Example: :: diff --git a/pythainlp/generate/thai2fit.py b/pythainlp/generate/thai2fit.py index 5e6455db9..3f93c9090 100644 --- a/pythainlp/generate/thai2fit.py +++ b/pythainlp/generate/thai2fit.py @@ -120,7 +120,7 @@ def gen_sentence( :param bool duplicate: allow duplicate words in sentence :return: list words or str words - :rtype: List[str], str + :rtype: list[str], str :Example: :: diff --git a/pythainlp/khavee/core.py b/pythainlp/khavee/core.py index 18135c245..06903775e 100644 --- a/pythainlp/khavee/core.py +++ b/pythainlp/khavee/core.py @@ -1,691 +1,691 @@ -# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project -# SPDX-FileType: SOURCE -# SPDX-License-Identifier: Apache-2.0 -# ruff: noqa: C901 -from __future__ import annotations - -from typing import Union - -from pythainlp import thai_consonants -from pythainlp.tokenize import subword_tokenize -from pythainlp.util import remove_tonemark, sound_syllable - - -class KhaveeVerifier: - def __init__(self): - """ - KhaveeVerifier: Thai Poetry verifier - """ - - def _has_true_final_yl(self, word: str) -> bool: - """ - Check if ย or ล is a true final consonant - (not just part of the vowel sound with ไ/ใ) - - :param str word: Thai word - :return: True if ย or ล is a true final consonant - :rtype: bool - """ - if len(word) < 2: - return False - # Count consonants in the word - consonant_count = sum(1 for c in word if c in thai_consonants) - # If there are 2+ consonants and word ends with ย or ล, it's a true final - return consonant_count >= 2 and word[-1] in ["ย", "ล"] - - def check_sara(self, word: str) -> str: - """ - Check the vowels in the Thai word. - - :param str word: Thai word - :return: vowel name of the word - :rtype: str - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.check_sara("เริง")) - # output: 'เออ' - """ - sara = [] - countoa = 0 - - # In case of การันย์ - if "์" in word[-1]: - word = word[:-2] - - # In case of สระเดี่ยว - for i in word: - if i in ("ะ", "ั"): - sara.append("อะ") - elif i == "ิ": - sara.append("อิ") - elif i == "ุ": - sara.append("อุ") - elif i == "ึ": - sara.append("อึ") - elif i == "ี": - sara.append("อี") - elif i == "ู": - sara.append("อู") - elif i == "ื": - sara.append("อือ") - elif i == "เ": - sara.append("เอ") - elif i == "แ": - sara.append("แอ") - elif i == "า": - sara.append("อา") - elif i == "โ": - sara.append("โอ") - elif i == "ำ": - sara.append("อำ") - elif i == "อ": - countoa += 1 - sara.append("ออ") - elif i == "ั" and "ว" in word: - sara.append("อัว") - elif i in ("ไ", "ใ"): - sara.append("ไอ") - elif i == "็": - sara.append("ออ") - elif "รร" in word: - if self.check_marttra(word) == "กม": - sara.append("อำ") - else: - sara.append("อะ") - - # In case of ออ - if countoa == 1 and "อ" in word[-1] and "เ" not in word: - sara.remove("ออ") - - # In case of เอ เอ - countA = 0 - for i in sara: - if i == "เอ": - countA = countA + 1 - if countA > 1: - sara.remove("เอ") - sara.remove("เอ") - sara.append("แ") - - # In case of สระประสม - if "เอ" in sara and "อะ" in sara: - sara.remove("เอ") - sara.remove("อะ") - sara.append("เอะ") - elif "แอ" in sara and "อะ" in sara: - sara.remove("แอ") - sara.remove("อะ") - sara.append("แอะ") - - if "เอะ" in sara and "ออ" in sara: - sara.remove("เอะ") - sara.remove("ออ") - sara.append("เออะ") - elif "เอ" in sara and "อิ" in sara: - sara.remove("เอ") - sara.remove("อิ") - sara.append("เออ") - elif "เอ" in sara and "ออ" in sara and "อ" in word[-1]: - sara.remove("เอ") - sara.remove("ออ") - sara.append("เออ") - elif "โอ" in sara and "อะ" in sara: - sara.remove("โอ") - sara.remove("อะ") - sara.append("โอะ") - elif "เอ" in sara and "อี" in sara: - sara.remove("เอ") - sara.remove("อี") - sara.append("เอีย") - elif "เอ" in sara and "อือ" in sara: - sara.remove("เอ") - sara.remove("อือ") - sara.append("อัว") - elif "เอ" in sara and "อา" in sara: - sara.remove("เอ") - sara.remove("อา") - sara.append("เอา") - elif "เ" in word and "า" in word and "ะ" in word: - sara = [] - sara.append("เอาะ") - - if "อือ" in sara and "เออ" in sara: - sara.remove("เออ") - sara.remove("อือ") - sara.append("เอือ") - elif "ออ" in sara and len(sara) > 1: - sara.remove("ออ") - elif "ว" in word and len(sara) == 0: - sara.append("อัว") - - if "ั" in word and self.check_marttra(word) == "กา": - sara = [] - sara.append("ไอ") - - # In case of อ - if word == "เออะ": - sara = [] - sara.append("เออะ") - elif word == "เออ": - sara = [] - sara.append("เออ") - elif word == "เอ": - sara = [] - sara.append("เอ") - elif word == "เอะ": - sara = [] - sara.append("เอะ") - elif word == "เอา": - sara = [] - sara.append("เอา") - elif word == "เอาะ": - sara = [] - sara.append("เอาะ") - - if "ฤา" in word or "ฦา" in word: - sara = [] - sara.append("อือ") - elif "ฤ" in word or "ฦ" in word: - sara = [] - sara.append("อึ") - - # In case of กน - if not sara and len(word) == 2: - if word[-1] != "ร": - sara.append("โอะ") - else: - sara.append("ออ") - elif not sara and len(word) == 3: - sara.append("ออ") - - # In case of บ่ - if word == "บ่": - sara = [] - sara.append("ออ") - - if "ํ" in word: - sara = [] - sara.append("อำ") - - if "เ" in word and "ื" in word and "อ" in word: - sara = [] - sara.append("เอือ") - - if not sara: - return "Can't find Sara in this word" - - return sara[0] - - def check_marttra(self, word: str) -> str: - """ - Check the Thai spelling Section in the Thai word. - - :param str word: Thai word - :return: name of spelling Section of the word. - :rtype: str - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.check_marttra("สาว")) - # output: 'เกอว' - """ - # Handle consonant clusters ending with ร - # ตร, ทร → remove ร (treat as final ต/ท sound) - # กร, ขร, คร, ฆร in compound words → remove ร (treat as final ก/ข/ค sound) - # But single syllable words like "กร" should keep ร - if len(word) >= 3 and word[-1] == "ร": - if word[-2] in ["ต", "ท"]: - word = word[:-1] - elif word[-2] in ["ก", "ข", "ค", "ฆ"]: - word = word[:-1] - - word = self.handle_karun_sound_silence(word) - word = remove_tonemark(word) - - # Check for ำ at the end (represents "am" sound, ends with m) - if word[-1] == "ำ": - return "กม" - - # Check for vowels and special patterns that indicate open syllables (กา) - # For words with ไ/ใ, check if ย/ล is a true final or just part of vowel - if "ไ" in word or "ใ" in word: - if word[-1] not in ["ย", "ล"]: - return "กา" - elif not self._has_true_final_yl(word): - # ย/ล is part of the vowel sound, not a true final - return "กา" - # else: ย/ล is a true final, continue to consonant classification below - - if "ํ" in word and "า" in word: - return "กา" - elif ( - word[-1] in ["า", "ะ", "ิ", "ี", "ุ", "ู", "อ"] - or ("ี" in word and "ย" in word[-1]) - or ("ื" in word and "อ" in word[-1]) - ): - return "กา" - elif word[-1] in ["ง"]: - return "กง" - elif word[-1] in ["ม"]: - return "กม" - elif word[-1] in ["ย"]: - return "เกย" - elif word[-1] in ["ล"]: - return "เกย" - elif word[-1] in ["ว"]: - return "เกอว" - elif word[-1] in ["ก", "ข", "ค", "ฆ"]: - return "กก" - elif word[-1] in [ - "จ", - "ช", - "ซ", - "ฎ", - "ฏ", - "ฐ", - "ฑ", - "ฒ", - "ด", - "ต", - "ถ", - "ท", - "ธ", - "ศ", - "ษ", - "ส", - ]: - return "กด" - elif word[-1] in ["ญ", "ณ", "น", "ร", "ฬ"]: - return "กน" - elif word[-1] in ["บ", "ป", "พ", "ฟ", "ภ"]: - return "กบ" - else: - if "็" in word: - return "กา" - else: - return "Can't find Marttra in this word" - - def is_sumpus(self, word1: str, word2: str) -> bool: - """ - Check the rhyme between two words. - - :param str word1: Thai word - :param str word2: Thai word - :return: boolean - :rtype: bool - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.is_sumpus("สรร", "อัน")) - # output: True - - print(kv.is_sumpus("สรร", "แมว")) - # output: False - """ - marttra1 = self.check_marttra(word1) - marttra2 = self.check_marttra(word2) - sara1 = self.check_sara(word1) - sara2 = self.check_sara(word2) - if sara1 == "อะ" and marttra1 == "เกย": - sara1 = "ไอ" - marttra1 = "กา" - elif sara2 == "อะ" and marttra2 == "เกย": - sara2 = "ไอ" - marttra2 = "กา" - if sara1 == "อำ" and marttra1 == "กม": - sara1 = "อำ" - marttra1 = "กา" - elif sara2 == "อำ" and marttra2 == "กม": - sara2 = "อำ" - marttra2 = "กา" - return bool(marttra1 == marttra2 and sara1 == sara2) - - def check_karu_lahu(self, text): - if ( - self.check_marttra(text) != "กา" - or ( - self.check_marttra(text) == "กา" - and self.check_sara(text) - in [ - "อา", - "อี", - "อือ", - "อู", - "เอ", - "แอ", - "โอ", - "ออ", - "เออ", - "เอีย", - "เอือ", - "อัว", - ] - ) - or self.check_sara(text) in ["อำ", "ไอ", "เอา"] - ) and text not in ["บ่", "ณ", "ธ", "ก็"]: - return "karu" - else: - return "lahu" - - def check_klon(self, text: str, k_type: int = 8) -> Union[list[str], str]: - """ - Check the suitability of the poem according to Thai principles. - - :param str text: Thai poem - :param int k_type: type of Thai poem - :return: the check results of the suitability of the - poem according to Thai principles. - :rtype: Union[List[str], str] - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - print(kv.check_klon( - 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง เอ๋งเอ๋งคะนอง \ - มีคนจับจอง เขาชื่อน้องเธียร', - k_type=4 - )) - # output: The poem is correct according to the principle. - - print(kv.check_klon( - 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง \ - เอ๋งเอ๋งเสียงหมา มีคนจับจอง เขาชื่อน้องเธียร', - k_type=4 - )) - # output: [ - "Can't find rhyme between paragraphs ('หมา', 'จอง') in paragraph 2", - "Can't find rhyme between paragraphs ('หมา', 'ทอง') in paragraph 2" - ] - """ - if k_type == 8: - try: - error = [] - list_sumpus_sent1 = [] - list_sumpus_sent2h = [] - list_sumpus_sent2l = [] - list_sumpus_sent3 = [] - list_sumpus_sent4 = [] - for i, sent in enumerate(text.split()): - sub_sent = subword_tokenize(sent, engine="dict") - if len(sub_sent) > 10: - error.append( - "In sentence " - + str(i + 2) - + ", there are more than 10 words. " - + str(sub_sent) - ) - if (i + 1) % 4 == 1: - list_sumpus_sent1.append(sub_sent[-1]) - elif (i + 1) % 4 == 2: - list_sumpus_sent2h.append( - [ - sub_sent[1], - sub_sent[2], - sub_sent[3], - sub_sent[4], - ] - ) - list_sumpus_sent2l.append(sub_sent[-1]) - elif (i + 1) % 4 == 3: - list_sumpus_sent3.append(sub_sent[-1]) - elif (i + 1) % 4 == 0: - list_sumpus_sent4.append(sub_sent[-1]) - if ( - len(list_sumpus_sent1) != len(list_sumpus_sent2h) - or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) - or len(list_sumpus_sent2l) != len(list_sumpus_sent3) - or len(list_sumpus_sent3) != len(list_sumpus_sent4) - or len(list_sumpus_sent4) != len(list_sumpus_sent1) - ): - return "The poem does not have 4 complete sentences." - else: - for i in range(len(list_sumpus_sent1)): - countwrong = 0 - for j in list_sumpus_sent2h[i]: - if ( - self.is_sumpus(list_sumpus_sent1[i], j) - is False - ): - countwrong += 1 - if countwrong > 3: - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent1[i], - list_sumpus_sent2h[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if ( - self.is_sumpus( - list_sumpus_sent2l[i], list_sumpus_sent3[i] - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent3[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if i > 0: - if ( - self.is_sumpus( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if not error: - return ( - "The poem is correct according to the principle." - ) - else: - return error - except: - return "Something went wrong. Make sure you enter it in the correct form of klon 8." - elif k_type == 4: - try: - error = [] - list_sumpus_sent1 = [] - list_sumpus_sent2h = [] - list_sumpus_sent2l = [] - list_sumpus_sent3 = [] - list_sumpus_sent4 = [] - for i, sent in enumerate(text.split()): - sub_sent = subword_tokenize(sent, engine="dict") - if len(sub_sent) > 5: - error.append( - "In sentence " - + str(i + 2) - + ", there are more than 4 words. " - + str(sub_sent) - ) - if (i + 1) % 4 == 1: - list_sumpus_sent1.append(sub_sent[-1]) - elif (i + 1) % 4 == 2: - list_sumpus_sent2h.append([sub_sent[1], sub_sent[2]]) - list_sumpus_sent2l.append(sub_sent[-1]) - elif (i + 1) % 4 == 3: - list_sumpus_sent3.append(sub_sent[-1]) - elif (i + 1) % 4 == 0: - list_sumpus_sent4.append(sub_sent[-1]) - if ( - len(list_sumpus_sent1) != len(list_sumpus_sent2h) - or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) - or len(list_sumpus_sent2l) != len(list_sumpus_sent3) - or len(list_sumpus_sent3) != len(list_sumpus_sent4) - or len(list_sumpus_sent4) != len(list_sumpus_sent1) - ): - return "The poem does not have 4 complete sentences." - else: - for i in range(len(list_sumpus_sent1)): - countwrong = 0 - for j in list_sumpus_sent2h[i]: - if ( - self.is_sumpus(list_sumpus_sent1[i], j) - is False - ): - countwrong += 1 - if countwrong > 1: - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent1[i], - list_sumpus_sent2h[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if ( - self.is_sumpus( - list_sumpus_sent2l[i], list_sumpus_sent3[i] - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent3[i], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if i > 0: - if ( - self.is_sumpus( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - is False - ): - error.append( - "Can't find rhyme between paragraphs " - + str( - ( - list_sumpus_sent2l[i], - list_sumpus_sent4[i - 1], - ) - ) - + " in paragraph " - + str(i + 1) - ) - if not error: - return ( - "The poem is correct according to the principle." - ) - else: - return error - except: - return "Something went wrong. Make sure you enter it in the correct form." - - else: - return "Something went wrong. Make sure you enter it in the correct form." - - def check_aek_too( - self, text: Union[list[str], str], dead_syllable_as_aek: bool = False - ) -> Union[list[Union[bool, str]], bool, str]: - """ - Checker of Thai tonal words - - :param Union[List[str], str] text: Thai word or list of Thai words - :param bool dead_syllable_as_aek: if True, dead syllable will - be considered as aek - :return: the check result if the word is aek or too - or False (not both) or list of check results if input is list - :rtype: Union[List[bool], List[str], bool, str] - - :Example: - :: - - from pythainlp.khavee import KhaveeVerifier - - kv = KhaveeVerifier() - - # การเช็คคำเอกโท - print( - kv.check_aek_too("เอง"), - kv.check_aek_too("เอ่ง"), - kv.check_aek_too("เอ้ง"), - ) - # -> False, aek, too - print(kv.check_aek_too(["เอง", "เอ่ง", "เอ้ง"])) # ใช้ List ได้เหมือนกัน - # -> [False, 'aek', 'too'] - """ - if isinstance(text, list): - return [self.check_aek_too(t, dead_syllable_as_aek) for t in text] # type: ignore[misc] - - if not isinstance(text, str): - raise TypeError("text must be str or iterable list[str]") - - word_characters = [*text] - if "่" in word_characters and "้" not in word_characters: - return "aek" - elif "้" in word_characters and "่" not in word_characters: - return "too" - if dead_syllable_as_aek and sound_syllable(text) == "dead": - return "aek" - else: - return False - - def handle_karun_sound_silence(self, word: str) -> str: - """ - Handle silent sounds in Thai words using '์' character (Karun) - by stripping all characters before the 'Karun' character - that should be silenced - - :param str text: Thai word - :return: Thai word with silent words stripped - :rtype: str - """ - sound_silenced = word.endswith("์") - if not sound_silenced: - return word - # Remove ์ and the silent consonant before it - # การันต์ (์) marks the consonant immediately before it as silent - word = word[:-2] - return word +# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project +# SPDX-FileType: SOURCE +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: C901 +from __future__ import annotations + +from typing import Union + +from pythainlp import thai_consonants +from pythainlp.tokenize import subword_tokenize +from pythainlp.util import remove_tonemark, sound_syllable + + +class KhaveeVerifier: + def __init__(self): + """ + KhaveeVerifier: Thai Poetry verifier + """ + + def _has_true_final_yl(self, word: str) -> bool: + """ + Check if ย or ล is a true final consonant + (not just part of the vowel sound with ไ/ใ) + + :param str word: Thai word + :return: True if ย or ล is a true final consonant + :rtype: bool + """ + if len(word) < 2: + return False + # Count consonants in the word + consonant_count = sum(1 for c in word if c in thai_consonants) + # If there are 2+ consonants and word ends with ย or ล, it's a true final + return consonant_count >= 2 and word[-1] in ["ย", "ล"] + + def check_sara(self, word: str) -> str: + """ + Check the vowels in the Thai word. + + :param str word: Thai word + :return: vowel name of the word + :rtype: str + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.check_sara("เริง")) + # output: 'เออ' + """ + sara = [] + countoa = 0 + + # In case of การันย์ + if "์" in word[-1]: + word = word[:-2] + + # In case of สระเดี่ยว + for i in word: + if i in ("ะ", "ั"): + sara.append("อะ") + elif i == "ิ": + sara.append("อิ") + elif i == "ุ": + sara.append("อุ") + elif i == "ึ": + sara.append("อึ") + elif i == "ี": + sara.append("อี") + elif i == "ู": + sara.append("อู") + elif i == "ื": + sara.append("อือ") + elif i == "เ": + sara.append("เอ") + elif i == "แ": + sara.append("แอ") + elif i == "า": + sara.append("อา") + elif i == "โ": + sara.append("โอ") + elif i == "ำ": + sara.append("อำ") + elif i == "อ": + countoa += 1 + sara.append("ออ") + elif i == "ั" and "ว" in word: + sara.append("อัว") + elif i in ("ไ", "ใ"): + sara.append("ไอ") + elif i == "็": + sara.append("ออ") + elif "รร" in word: + if self.check_marttra(word) == "กม": + sara.append("อำ") + else: + sara.append("อะ") + + # In case of ออ + if countoa == 1 and "อ" in word[-1] and "เ" not in word: + sara.remove("ออ") + + # In case of เอ เอ + countA = 0 + for i in sara: + if i == "เอ": + countA = countA + 1 + if countA > 1: + sara.remove("เอ") + sara.remove("เอ") + sara.append("แ") + + # In case of สระประสม + if "เอ" in sara and "อะ" in sara: + sara.remove("เอ") + sara.remove("อะ") + sara.append("เอะ") + elif "แอ" in sara and "อะ" in sara: + sara.remove("แอ") + sara.remove("อะ") + sara.append("แอะ") + + if "เอะ" in sara and "ออ" in sara: + sara.remove("เอะ") + sara.remove("ออ") + sara.append("เออะ") + elif "เอ" in sara and "อิ" in sara: + sara.remove("เอ") + sara.remove("อิ") + sara.append("เออ") + elif "เอ" in sara and "ออ" in sara and "อ" in word[-1]: + sara.remove("เอ") + sara.remove("ออ") + sara.append("เออ") + elif "โอ" in sara and "อะ" in sara: + sara.remove("โอ") + sara.remove("อะ") + sara.append("โอะ") + elif "เอ" in sara and "อี" in sara: + sara.remove("เอ") + sara.remove("อี") + sara.append("เอีย") + elif "เอ" in sara and "อือ" in sara: + sara.remove("เอ") + sara.remove("อือ") + sara.append("อัว") + elif "เอ" in sara and "อา" in sara: + sara.remove("เอ") + sara.remove("อา") + sara.append("เอา") + elif "เ" in word and "า" in word and "ะ" in word: + sara = [] + sara.append("เอาะ") + + if "อือ" in sara and "เออ" in sara: + sara.remove("เออ") + sara.remove("อือ") + sara.append("เอือ") + elif "ออ" in sara and len(sara) > 1: + sara.remove("ออ") + elif "ว" in word and len(sara) == 0: + sara.append("อัว") + + if "ั" in word and self.check_marttra(word) == "กา": + sara = [] + sara.append("ไอ") + + # In case of อ + if word == "เออะ": + sara = [] + sara.append("เออะ") + elif word == "เออ": + sara = [] + sara.append("เออ") + elif word == "เอ": + sara = [] + sara.append("เอ") + elif word == "เอะ": + sara = [] + sara.append("เอะ") + elif word == "เอา": + sara = [] + sara.append("เอา") + elif word == "เอาะ": + sara = [] + sara.append("เอาะ") + + if "ฤา" in word or "ฦา" in word: + sara = [] + sara.append("อือ") + elif "ฤ" in word or "ฦ" in word: + sara = [] + sara.append("อึ") + + # In case of กน + if not sara and len(word) == 2: + if word[-1] != "ร": + sara.append("โอะ") + else: + sara.append("ออ") + elif not sara and len(word) == 3: + sara.append("ออ") + + # In case of บ่ + if word == "บ่": + sara = [] + sara.append("ออ") + + if "ํ" in word: + sara = [] + sara.append("อำ") + + if "เ" in word and "ื" in word and "อ" in word: + sara = [] + sara.append("เอือ") + + if not sara: + return "Can't find Sara in this word" + + return sara[0] + + def check_marttra(self, word: str) -> str: + """ + Check the Thai spelling Section in the Thai word. + + :param str word: Thai word + :return: name of spelling Section of the word. + :rtype: str + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.check_marttra("สาว")) + # output: 'เกอว' + """ + # Handle consonant clusters ending with ร + # ตร, ทร → remove ร (treat as final ต/ท sound) + # กร, ขร, คร, ฆร in compound words → remove ร (treat as final ก/ข/ค sound) + # But single syllable words like "กร" should keep ร + if len(word) >= 3 and word[-1] == "ร": + if word[-2] in ["ต", "ท"]: + word = word[:-1] + elif word[-2] in ["ก", "ข", "ค", "ฆ"]: + word = word[:-1] + + word = self.handle_karun_sound_silence(word) + word = remove_tonemark(word) + + # Check for ำ at the end (represents "am" sound, ends with m) + if word[-1] == "ำ": + return "กม" + + # Check for vowels and special patterns that indicate open syllables (กา) + # For words with ไ/ใ, check if ย/ล is a true final or just part of vowel + if "ไ" in word or "ใ" in word: + if word[-1] not in ["ย", "ล"]: + return "กา" + elif not self._has_true_final_yl(word): + # ย/ล is part of the vowel sound, not a true final + return "กา" + # else: ย/ล is a true final, continue to consonant classification below + + if "ํ" in word and "า" in word: + return "กา" + elif ( + word[-1] in ["า", "ะ", "ิ", "ี", "ุ", "ู", "อ"] + or ("ี" in word and "ย" in word[-1]) + or ("ื" in word and "อ" in word[-1]) + ): + return "กา" + elif word[-1] in ["ง"]: + return "กง" + elif word[-1] in ["ม"]: + return "กม" + elif word[-1] in ["ย"]: + return "เกย" + elif word[-1] in ["ล"]: + return "เกย" + elif word[-1] in ["ว"]: + return "เกอว" + elif word[-1] in ["ก", "ข", "ค", "ฆ"]: + return "กก" + elif word[-1] in [ + "จ", + "ช", + "ซ", + "ฎ", + "ฏ", + "ฐ", + "ฑ", + "ฒ", + "ด", + "ต", + "ถ", + "ท", + "ธ", + "ศ", + "ษ", + "ส", + ]: + return "กด" + elif word[-1] in ["ญ", "ณ", "น", "ร", "ฬ"]: + return "กน" + elif word[-1] in ["บ", "ป", "พ", "ฟ", "ภ"]: + return "กบ" + else: + if "็" in word: + return "กา" + else: + return "Can't find Marttra in this word" + + def is_sumpus(self, word1: str, word2: str) -> bool: + """ + Check the rhyme between two words. + + :param str word1: Thai word + :param str word2: Thai word + :return: boolean + :rtype: bool + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.is_sumpus("สรร", "อัน")) + # output: True + + print(kv.is_sumpus("สรร", "แมว")) + # output: False + """ + marttra1 = self.check_marttra(word1) + marttra2 = self.check_marttra(word2) + sara1 = self.check_sara(word1) + sara2 = self.check_sara(word2) + if sara1 == "อะ" and marttra1 == "เกย": + sara1 = "ไอ" + marttra1 = "กา" + elif sara2 == "อะ" and marttra2 == "เกย": + sara2 = "ไอ" + marttra2 = "กา" + if sara1 == "อำ" and marttra1 == "กม": + sara1 = "อำ" + marttra1 = "กา" + elif sara2 == "อำ" and marttra2 == "กม": + sara2 = "อำ" + marttra2 = "กา" + return bool(marttra1 == marttra2 and sara1 == sara2) + + def check_karu_lahu(self, text): + if ( + self.check_marttra(text) != "กา" + or ( + self.check_marttra(text) == "กา" + and self.check_sara(text) + in [ + "อา", + "อี", + "อือ", + "อู", + "เอ", + "แอ", + "โอ", + "ออ", + "เออ", + "เอีย", + "เอือ", + "อัว", + ] + ) + or self.check_sara(text) in ["อำ", "ไอ", "เอา"] + ) and text not in ["บ่", "ณ", "ธ", "ก็"]: + return "karu" + else: + return "lahu" + + def check_klon(self, text: str, k_type: int = 8) -> Union[list[str], str]: + """ + Check the suitability of the poem according to Thai principles. + + :param str text: Thai poem + :param int k_type: type of Thai poem + :return: the check results of the suitability of the + poem according to Thai principles. + :rtype: Union[list[str], str] + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + print(kv.check_klon( + 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง เอ๋งเอ๋งคะนอง \ + มีคนจับจอง เขาชื่อน้องเธียร', + k_type=4 + )) + # output: The poem is correct according to the principle. + + print(kv.check_klon( + 'ฉันชื่อหมูกรอบ ฉันชอบกินไก่ แล้วก็วิ่งไล่ หมาชื่อนํ้าทอง ลคคนเก่ง \ + เอ๋งเอ๋งเสียงหมา มีคนจับจอง เขาชื่อน้องเธียร', + k_type=4 + )) + # output: [ + "Can't find rhyme between paragraphs ('หมา', 'จอง') in paragraph 2", + "Can't find rhyme between paragraphs ('หมา', 'ทอง') in paragraph 2" + ] + """ + if k_type == 8: + try: + error = [] + list_sumpus_sent1 = [] + list_sumpus_sent2h = [] + list_sumpus_sent2l = [] + list_sumpus_sent3 = [] + list_sumpus_sent4 = [] + for i, sent in enumerate(text.split()): + sub_sent = subword_tokenize(sent, engine="dict") + if len(sub_sent) > 10: + error.append( + "In sentence " + + str(i + 2) + + ", there are more than 10 words. " + + str(sub_sent) + ) + if (i + 1) % 4 == 1: + list_sumpus_sent1.append(sub_sent[-1]) + elif (i + 1) % 4 == 2: + list_sumpus_sent2h.append( + [ + sub_sent[1], + sub_sent[2], + sub_sent[3], + sub_sent[4], + ] + ) + list_sumpus_sent2l.append(sub_sent[-1]) + elif (i + 1) % 4 == 3: + list_sumpus_sent3.append(sub_sent[-1]) + elif (i + 1) % 4 == 0: + list_sumpus_sent4.append(sub_sent[-1]) + if ( + len(list_sumpus_sent1) != len(list_sumpus_sent2h) + or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) + or len(list_sumpus_sent2l) != len(list_sumpus_sent3) + or len(list_sumpus_sent3) != len(list_sumpus_sent4) + or len(list_sumpus_sent4) != len(list_sumpus_sent1) + ): + return "The poem does not have 4 complete sentences." + else: + for i in range(len(list_sumpus_sent1)): + countwrong = 0 + for j in list_sumpus_sent2h[i]: + if ( + self.is_sumpus(list_sumpus_sent1[i], j) + is False + ): + countwrong += 1 + if countwrong > 3: + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent1[i], + list_sumpus_sent2h[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if ( + self.is_sumpus( + list_sumpus_sent2l[i], list_sumpus_sent3[i] + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent3[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if i > 0: + if ( + self.is_sumpus( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if not error: + return ( + "The poem is correct according to the principle." + ) + else: + return error + except: + return "Something went wrong. Make sure you enter it in the correct form of klon 8." + elif k_type == 4: + try: + error = [] + list_sumpus_sent1 = [] + list_sumpus_sent2h = [] + list_sumpus_sent2l = [] + list_sumpus_sent3 = [] + list_sumpus_sent4 = [] + for i, sent in enumerate(text.split()): + sub_sent = subword_tokenize(sent, engine="dict") + if len(sub_sent) > 5: + error.append( + "In sentence " + + str(i + 2) + + ", there are more than 4 words. " + + str(sub_sent) + ) + if (i + 1) % 4 == 1: + list_sumpus_sent1.append(sub_sent[-1]) + elif (i + 1) % 4 == 2: + list_sumpus_sent2h.append([sub_sent[1], sub_sent[2]]) + list_sumpus_sent2l.append(sub_sent[-1]) + elif (i + 1) % 4 == 3: + list_sumpus_sent3.append(sub_sent[-1]) + elif (i + 1) % 4 == 0: + list_sumpus_sent4.append(sub_sent[-1]) + if ( + len(list_sumpus_sent1) != len(list_sumpus_sent2h) + or len(list_sumpus_sent2h) != len(list_sumpus_sent2l) + or len(list_sumpus_sent2l) != len(list_sumpus_sent3) + or len(list_sumpus_sent3) != len(list_sumpus_sent4) + or len(list_sumpus_sent4) != len(list_sumpus_sent1) + ): + return "The poem does not have 4 complete sentences." + else: + for i in range(len(list_sumpus_sent1)): + countwrong = 0 + for j in list_sumpus_sent2h[i]: + if ( + self.is_sumpus(list_sumpus_sent1[i], j) + is False + ): + countwrong += 1 + if countwrong > 1: + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent1[i], + list_sumpus_sent2h[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if ( + self.is_sumpus( + list_sumpus_sent2l[i], list_sumpus_sent3[i] + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent3[i], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if i > 0: + if ( + self.is_sumpus( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + is False + ): + error.append( + "Can't find rhyme between paragraphs " + + str( + ( + list_sumpus_sent2l[i], + list_sumpus_sent4[i - 1], + ) + ) + + " in paragraph " + + str(i + 1) + ) + if not error: + return ( + "The poem is correct according to the principle." + ) + else: + return error + except: + return "Something went wrong. Make sure you enter it in the correct form." + + else: + return "Something went wrong. Make sure you enter it in the correct form." + + def check_aek_too( + self, text: Union[list[str], str], dead_syllable_as_aek: bool = False + ) -> Union[list[Union[bool, str]], bool, str]: + """ + Checker of Thai tonal words + + :param Union[list[str], str] text: Thai word or list of Thai words + :param bool dead_syllable_as_aek: if True, dead syllable will + be considered as aek + :return: the check result if the word is aek or too + or False (not both) or list of check results if input is list + :rtype: Union[list[bool], List[str], bool, str] + + :Example: + :: + + from pythainlp.khavee import KhaveeVerifier + + kv = KhaveeVerifier() + + # การเช็คคำเอกโท + print( + kv.check_aek_too("เอง"), + kv.check_aek_too("เอ่ง"), + kv.check_aek_too("เอ้ง"), + ) + # -> False, aek, too + print(kv.check_aek_too(["เอง", "เอ่ง", "เอ้ง"])) # ใช้ List ได้เหมือนกัน + # -> [False, 'aek', 'too'] + """ + if isinstance(text, list): + return [self.check_aek_too(t, dead_syllable_as_aek) for t in text] # type: ignore[misc] + + if not isinstance(text, str): + raise TypeError("text must be str or iterable list[str]") + + word_characters = [*text] + if "่" in word_characters and "้" not in word_characters: + return "aek" + elif "้" in word_characters and "่" not in word_characters: + return "too" + if dead_syllable_as_aek and sound_syllable(text) == "dead": + return "aek" + else: + return False + + def handle_karun_sound_silence(self, word: str) -> str: + """ + Handle silent sounds in Thai words using '์' character (Karun) + by stripping all characters before the 'Karun' character + that should be silenced + + :param str text: Thai word + :return: Thai word with silent words stripped + :rtype: str + """ + sound_silenced = word.endswith("์") + if not sound_silenced: + return word + # Remove ์ and the silent consonant before it + # การันต์ (์) marks the consonant immediately before it as silent + word = word[:-2] + return word diff --git a/pythainlp/lm/text_util.py b/pythainlp/lm/text_util.py index 51d16939d..1e6cd6b9d 100644 --- a/pythainlp/lm/text_util.py +++ b/pythainlp/lm/text_util.py @@ -15,7 +15,7 @@ def calculate_ngram_counts( :param int n_max: The maximum n-gram size (default: 4). :return: A dictionary where keys are n-grams and values are their counts. - :rtype: Dict[Tuple[str], int] + :rtype: dict[tuple[str, ...], int] """ if not list_words: return {} @@ -36,7 +36,7 @@ def remove_repeated_ngrams(string_list: list[str], n: int = 2) -> list[str]: :param List[str] string_list: List of string :param int n: n-gram size :return: List of string - :rtype: List[str] + :rtype: list[str] :Example: :: diff --git a/pythainlp/phayathaibert/core.py b/pythainlp/phayathaibert/core.py index 176ba833c..f622b2916 100644 --- a/pythainlp/phayathaibert/core.py +++ b/pythainlp/phayathaibert/core.py @@ -136,9 +136,9 @@ def _replace_rep(m): def replace_wrep_post(self, toks: list[str]) -> list[str]: """Replace repetitive words post tokenization; fastai `replace_wrep` does not work well with Thai. - :param List[str] toks: list of tokens + :param list[str] toks: list of tokens :return: list of tokens where repetitive words are removed. - :rtype: List[str] + :rtype: list[str] :Example: >>> toks = ["กา", "น้ำ", "น้ำ", "น้ำ", "น้ำ"] >>> replace_wrep_post(toks) @@ -161,9 +161,9 @@ def replace_wrep_post(self, toks: list[str]) -> list[str]: def remove_space(self, toks: list[str]) -> list[str]: """Do not include space for bag-of-word models. - :param List[str] toks: list of tokens + :param list[str] toks: list of tokens :return: List of tokens where space tokens (" ") are filtered out - :rtype: List[str] + :rtype: list[str] :Example: >>> toks = ["ฉัน", "เดิน", " ", "กลับ", "บ้าน"] >>> remove_space(toks) @@ -256,7 +256,7 @@ def augment( true if more word diversity is needed :return: list of text augment - :rtype: List[str] + :rtype: list[str] :Example: :: @@ -367,7 +367,7 @@ def get_ner( specified as `True`). Otherwise, return a list of tuples associated with tokenized words and NER tags - :rtype: Union[List[Tuple[str, str]], List[Tuple[str, str, str]], str] + :rtype: Union[list[tuple[str, str]], list[tuple[str, str, str]], str] :Example: >>> from pythainlp.phayathaibert.core import NamedEntityTagger diff --git a/pythainlp/soundex/sound.py b/pythainlp/soundex/sound.py index 6977d3f85..f5065f6a9 100644 --- a/pythainlp/soundex/sound.py +++ b/pythainlp/soundex/sound.py @@ -61,7 +61,7 @@ def audio_vector(word: str) -> list[list[int]]: :param str word: Thai word :return: List of features from panphon - :rtype: List[List[int]] + :rtype: list[list[int]] :Example: :: @@ -80,7 +80,7 @@ def word_approximation(word: str, list_word: list[str]) -> list[float]: :param str word: Thai word :param str list_word: Thai word :return: List of approximation of words (The smaller the value, the closer) - :rtype: List[float] + :rtype: list[float] :Example: :: diff --git a/pythainlp/spell/core.py b/pythainlp/spell/core.py index d32c784fa..9cfc87723 100644 --- a/pythainlp/spell/core.py +++ b/pythainlp/spell/core.py @@ -142,13 +142,13 @@ def correct(word: str, engine: str = "pn") -> str: def spell_sent(list_words: list[str], engine: str = "pn") -> list[list[str]]: """Provides a list of possible correct spellings of sentence - :param List[str] list_words: list of words in sentence + :param list[str] list_words: list of words in sentence :param str engine: * *pn* - Peter Norvig's algorithm [#norvig_spellchecker]_ (default) * *phunspell* - A spell checker utilizing spylls, a port of Hunspell. * *symspellpy* - symspellpy is a Python port of SymSpell v6.5. :return: list of possibly correct words - :rtype: List[List[str]] + :rtype: list[list[str]] :Example: :: @@ -179,14 +179,14 @@ def spell_sent(list_words: list[str], engine: str = "pn") -> list[list[str]]: def correct_sent(list_words: list[str], engine: str = "pn") -> list[str]: """Corrects and returns the spelling of the given sentence - :param List[str] list_words: list of words in sentence + :param list[str] list_words: list of words in sentence :param str engine: * *pn* - Peter Norvig's algorithm [#norvig_spellchecker]_ (default) * *phunspell* - A spell checker utilizing spylls, a port of Hunspell. * *symspellpy* - symspellpy is a Python port of SymSpell v6.5. * *wanchanberta_thai_grammarly* - WanchanBERTa Thai Grammarly :return: the corrected list of words in sentence - :rtype: List[str] + :rtype: list[str] :Example: :: diff --git a/pythainlp/spell/pn.py b/pythainlp/spell/pn.py index f869fd6bb..c7db683e2 100644 --- a/pythainlp/spell/pn.py +++ b/pythainlp/spell/pn.py @@ -181,7 +181,7 @@ def dictionary(self) -> ItemsView[str, int]: """Returns the spelling dictionary currently used by this spell checker :return: spelling dictionary of this instance - :rtype: list[tuple[str, int]] + :rtype: ItemsView[str, int] :Example: :: diff --git a/pythainlp/spell/words_spelling_correction.py b/pythainlp/spell/words_spelling_correction.py index edbb9b854..d2f55d3bb 100644 --- a/pythainlp/spell/words_spelling_correction.py +++ b/pythainlp/spell/words_spelling_correction.py @@ -255,9 +255,9 @@ def get_words_spell_suggestion( Requirements: numpy and onnxruntime (Install before use this function) - :param Union[str, List[str]] list_word: list words or a word. + :param Union[str, list[str]] list_word: list words or a word. :return: List words spell suggestion (max 5 items per word) - :rtype: Union[List[str], List[List[str]]] + :rtype: Union[list[str], list[list[str]]] :Example: :: diff --git a/pythainlp/summarize/keybert.py b/pythainlp/summarize/keybert.py index ff7773d2e..bae3ed07e 100644 --- a/pythainlp/summarize/keybert.py +++ b/pythainlp/summarize/keybert.py @@ -162,15 +162,17 @@ def _generate_ngrams( tokenizer_engine: str, stop_words: Iterable[str], ) -> list[str]: - assert keyphrase_ngram_range[0] >= 1, ( - f"`keyphrase_ngram_range` must start from 1. " - f"current value={keyphrase_ngram_range}." - ) + if keyphrase_ngram_range[0] < 1: + raise ValueError( + f"`keyphrase_ngram_range` must start from 1. " + f"current value={keyphrase_ngram_range}." + ) - assert keyphrase_ngram_range[0] <= keyphrase_ngram_range[1], ( - f"The value first argument of `keyphrase_ngram_range` must not exceed the second. " - f"current value={keyphrase_ngram_range}." - ) + if keyphrase_ngram_range[0] > keyphrase_ngram_range[1]: + raise ValueError( + f"The value first argument of `keyphrase_ngram_range` must not exceed the second. " + f"current value={keyphrase_ngram_range}." + ) def _join_ngram(ngrams: list[tuple[str, ...]]) -> list[str]: ngrams_joined = [] @@ -217,9 +219,8 @@ def l2_norm(v: np.ndarray) -> np.ndarray: v, np.linalg.norm(v, axis=1).reshape(-1, 1).repeat(vec_size, axis=1), ) - assert np.isclose(np.linalg.norm(result, axis=1), 1).all(), ( - "Cannot normalize a vector to unit vector." - ) + if not np.isclose(np.linalg.norm(result, axis=1), 1).all(): + raise ValueError("Cannot normalize a vector to unit vector.") return result # type: ignore[no-any-return] def cosine_sim(a: np.ndarray, b: np.ndarray) -> np.ndarray: diff --git a/pythainlp/tag/chunk.py b/pythainlp/tag/chunk.py index 278da56a9..dc53e6f21 100644 --- a/pythainlp/tag/chunk.py +++ b/pythainlp/tag/chunk.py @@ -9,12 +9,12 @@ def chunk_parse( ) -> list[str]: """This function parses Thai sentence to phrase structure in IOB format. - :param list sent: list [(word, part-of-speech)] + :param list[tuple[str, str]] sent: list [(word, part-of-speech)] :param str engine: chunk parse engine (now, it has crf only) :param str corpus: chunk parse corpus (now, it has orchidpp only) :return: a list of tuples (word, part-of-speech, chunking) - :rtype: List[str] + :rtype: list[str] :Example: :: diff --git a/pythainlp/tag/named_entity.py b/pythainlp/tag/named_entity.py index 741532ab5..daa7cdcc0 100644 --- a/pythainlp/tag/named_entity.py +++ b/pythainlp/tag/named_entity.py @@ -84,7 +84,7 @@ def tag( specified as `True`). Otherwise, return a list of tuples associated with tokenized words and NER tags - :rtype: Union[List[Tuple[str, str]], List[Tuple[str, str, str]], str] + :rtype: Union[list[tuple[str, str]], list[tuple[str, str, str]], str] :Example: >>> from pythainlp.tag import NER @@ -129,7 +129,7 @@ def tag(self, text: str) -> tuple[list[str], list[dict[str, Any]]]: :param str text: text in Thai to be tagged :return: a list of tuples associated with tokenized words and NNER tags. - :rtype: Tuple[List[str], List[dict]] + :rtype: tuple[list[str], list[dict[str, Any]]] :Example: diff --git a/pythainlp/tag/thainer.py b/pythainlp/tag/thainer.py index 081de29cf..5f775cb98 100644 --- a/pythainlp/tag/thainer.py +++ b/pythainlp/tag/thainer.py @@ -136,7 +136,7 @@ def get_ner( specified as `True`). Otherwise, return a list of tuples associated with tokenized words and NER tags - :rtype: Union[list[tuple[str, str]], list[tuple[str, str, str]]], str + :rtype: Union[list[tuple[str, str]], list[tuple[str, str, str]], str] :Note: * For the POS tags to be included in the results, this function diff --git a/pythainlp/tag/tltk.py b/pythainlp/tag/tltk.py index 6a83bd1f0..3bbe0ec44 100644 --- a/pythainlp/tag/tltk.py +++ b/pythainlp/tag/tltk.py @@ -44,7 +44,7 @@ def get_ner( specified as `True`). Otherwise, return a list of tuples associated with tokenized words and NER tags - :rtype: Union[list[tuple[str, str]], list[tuple[str, str, str]]], str + :rtype: Union[list[tuple[str, str]], list[tuple[str, str, str]], str] :Example: diff --git a/pythainlp/tokenize/_utils.py b/pythainlp/tokenize/_utils.py index 6fa67de17..e5815ad24 100644 --- a/pythainlp/tokenize/_utils.py +++ b/pythainlp/tokenize/_utils.py @@ -27,9 +27,9 @@ def rejoin_formatted_num(segments: list[str]) -> list[str]: The formatted numeric are numbers separated by ":", ",", or ".", such as time, decimal numbers, comma-added numbers, and IP addresses. - :param List[str] segments: result from word tokenizer + :param list[str] segments: result from word tokenizer :return: a list of fixed tokens - :rtype: List[str] + :rtype: list[str] :Example: tokens = ['ขณะ', 'นี้', 'เวลา', ' ', '12', ':', '00น', ' ', 'อัตรา', @@ -72,9 +72,9 @@ def rejoin_formatted_num(segments: list[str]) -> list[str]: def strip_whitespace(segments: list[str]) -> list[str]: """Strip whitespace(s) off each token and remove whitespace tokens. - :param List[str] segments: result from word tokenizer + :param list[str] segments: result from word tokenizer :return: a list of tokens - :rtype: List[str] + :rtype: list[str] :Example: tokens = [" ", "วันนี้ ", "เวลา ", "19.00น"] diff --git a/pythainlp/tokenize/core.py b/pythainlp/tokenize/core.py index 54ccb65a6..fbb8f9b14 100644 --- a/pythainlp/tokenize/core.py +++ b/pythainlp/tokenize/core.py @@ -39,7 +39,7 @@ def word_detokenize( :param str segments: List of sentences, each with a list of words. :param str output: the output type (str or list) :return: the Thai text - :rtype: Union[str,List[str]] + :rtype: Union[list[list[str]], str] :Example: :: @@ -118,7 +118,7 @@ def word_tokenize( Otherwise, formatted numeric could be wrongly separated. :return: list of words - :rtype: List[str] + :rtype: list[str] **Options for engine** * *attacut* - wrapper for `AttaCut `_., @@ -423,7 +423,7 @@ def sent_tokenize( :param str engine: choose among *'crfcut'*, *'whitespace'*, \ *'whitespace+newline'* :return: list of split sentences - :rtype: list[str] + :rtype: Union[list[str], list[list[str]]] **Options for engine** * *crfcut* - (default) split by CRF trained on TED dataset * *thaisum* - The implementation of sentence segmenter from \ @@ -583,7 +583,7 @@ def paragraph_tokenize( :param str text: text to be tokenized :param str engine: the name of paragraph tokenizer :return: list of paragraphs - :rtype: List[List[str]] + :rtype: list[List[str]] **Options for engine** * *wtp* - split by `wtpsplitaxe `_., \ It supports many sizes of models. You can use ``wtp`` to use mini model, \ @@ -662,7 +662,7 @@ def subword_tokenize( :param str engine: the name of subword tokenizer :param bool keep_whitespace: keep whitespace :return: list of subwords - :rtype: List[str] + :rtype: list[str] **Options for engine** * *dict* - newmm word tokenizer with a syllable dictionary * *etcc* - Enhanced Thai Character Cluster (Inrut et al. 2001) @@ -785,7 +785,7 @@ def syllable_tokenize( :param str engine: the name of syllable tokenizer :param bool keep_whitespace: keep whitespace :return: list of subwords - :rtype: List[str] + :rtype: list[str] **Options for engine** * *dict* - newmm word tokenizer with a syllable dictionary * *han_solo* - CRF syllable segmenter for Thai that can work in the \ @@ -824,7 +824,7 @@ def display_cell_tokenize(text: str) -> list[str]: :param str text: text to be tokenized :return: list of display cells - :rtype: List[str] + :rtype: list[str] :Example: Tokenize Thai text into display cells:: diff --git a/pythainlp/tokenize/multi_cut.py b/pythainlp/tokenize/multi_cut.py index d6d7d1940..2a3fec9e6 100644 --- a/pythainlp/tokenize/multi_cut.py +++ b/pythainlp/tokenize/multi_cut.py @@ -133,7 +133,7 @@ def segment(text: str, custom_dict: Optional[Trie] = None) -> list[str]: defaults to a Trie generated from pythainlp.corpus.thai_words :type custom_dict: Trie, optional :return: list of segmented tokens - :rtype: List[str] + :rtype: list[str] """ if not text or not isinstance(text, str): return [] @@ -155,7 +155,7 @@ def find_all_segment( defaults to word_dict_trie() :type custom_dict: Trie, optional :return: list of segment variations - :rtype: List[str] + :rtype: list[str] """ if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/newmm.py b/pythainlp/tokenize/newmm.py index 77711097c..bd93a6b3b 100644 --- a/pythainlp/tokenize/newmm.py +++ b/pythainlp/tokenize/newmm.py @@ -161,7 +161,7 @@ def segment( with many ambiguous breaking points, defaults to False :type safe_mode: bool, optional :return: list of tokens - :rtype: List[str] + :rtype: list[str] """ if not text or not isinstance(text, str): return [] diff --git a/pythainlp/tokenize/nlpo3.py b/pythainlp/tokenize/nlpo3.py index 12ff94b82..add0d3281 100644 --- a/pythainlp/tokenize/nlpo3.py +++ b/pythainlp/tokenize/nlpo3.py @@ -97,7 +97,7 @@ def segment( :param bool parallel_mode: Use multithread mode, defaults to False :return: list of tokens - :rtype: List[str] + :rtype: list[str] :See Also: * \ diff --git a/pythainlp/transliterate/thai2rom.py b/pythainlp/transliterate/thai2rom.py index 155814b9f..9480ee1b3 100644 --- a/pythainlp/transliterate/thai2rom.py +++ b/pythainlp/transliterate/thai2rom.py @@ -275,7 +275,11 @@ def __init__( self.target_end_token = target_end_token self.max_length = max_length - assert encoder.hidden_size == decoder.hidden_size + if encoder.hidden_size != decoder.hidden_size: + raise ValueError( + f"Encoder and decoder hidden sizes must match. " + f"Got encoder={encoder.hidden_size}, decoder={decoder.hidden_size}" + ) def create_mask(self, source_seq): mask = source_seq != self.pad_idx @@ -299,7 +303,10 @@ def forward( ) if target_seq is None: - assert teacher_forcing_ratio == 0, "Must be zero during inference" + if teacher_forcing_ratio != 0: + raise ValueError( + "teacher_forcing_ratio must be zero during inference" + ) inference = True else: inference = False diff --git a/pythainlp/transliterate/thaig2p.py b/pythainlp/transliterate/thaig2p.py index eb8614512..b7cb3323f 100644 --- a/pythainlp/transliterate/thaig2p.py +++ b/pythainlp/transliterate/thaig2p.py @@ -288,7 +288,11 @@ def __init__( self.target_end_token = target_end_token self.max_length = max_length - assert encoder.hidden_size == decoder.hidden_size + if encoder.hidden_size != decoder.hidden_size: + raise ValueError( + f"Encoder and decoder hidden sizes must match. " + f"Got encoder={encoder.hidden_size}, decoder={decoder.hidden_size}" + ) def create_mask(self, source_seq): mask = source_seq != self.pad_idx @@ -312,7 +316,10 @@ def forward( ) if target_seq is None: - assert teacher_forcing_ratio == 0, "Must be zero during inference" + if teacher_forcing_ratio != 0: + raise ValueError( + "teacher_forcing_ratio must be zero during inference" + ) inference = True else: inference = False diff --git a/pythainlp/transliterate/w2p.py b/pythainlp/transliterate/w2p.py index 24a153080..d9ac1650e 100644 --- a/pythainlp/transliterate/w2p.py +++ b/pythainlp/transliterate/w2p.py @@ -58,10 +58,16 @@ def __init__(self): if self.checkpoint is None: download(_MODEL_NAME, version="0.2") self.checkpoint = get_corpus_path(_MODEL_NAME) + if self.checkpoint is None: + raise RuntimeError( + f"Failed to download or locate {_MODEL_NAME} corpus" + ) self._load_variables() def _load_variables(self) -> None: - self.variables = np.load(self.checkpoint, allow_pickle=True) # type: ignore[arg-type] + if self.checkpoint is None: + raise RuntimeError("checkpoint path is not set") + self.variables = np.load(self.checkpoint, allow_pickle=True) # (29, 64). (len(graphemes), emb) self.enc_emb = self.variables.item().get("encoder.emb.weight") # (3*128, 64) diff --git a/pythainlp/ulmfit/core.py b/pythainlp/ulmfit/core.py index 7c3a98519..b0615dac7 100644 --- a/pythainlp/ulmfit/core.py +++ b/pythainlp/ulmfit/core.py @@ -109,7 +109,7 @@ def process_thai( :param list[func] post_rules: rules to apply after tokenizations :return: a list of cleaned tokenized texts - :rtype: list[str] + :rtype: Collection[str] :Note: diff --git a/pythainlp/word_vector/core.py b/pythainlp/word_vector/core.py index 33e81c77e..b3fb99a38 100644 --- a/pythainlp/word_vector/core.py +++ b/pythainlp/word_vector/core.py @@ -78,7 +78,7 @@ def doesnt_match(self, words: list[str]) -> str: in the list. We use the function :func:`doesnt_match` from :mod:`gensim`. - :param list words: a list of words + :param list[str] words: a list of words :raises KeyError: if there is any word in `positive` or `negative` that is not in the vocabulary of the model. :return: the word that is mostly unrelated @@ -122,8 +122,8 @@ def most_similar_cosmul( We use the function :func:`gensim.most_similar_cosmul` directly from :mod:`gensim`. - :param list positive: a list of words to add - :param list negative: a list of words to subtract + :param list[str] positive: a list of words to add + :param list[str] negative: a list of words to subtract :raises KeyError: if there is any word in `positive` or `negative` that is not in the vocabulary of the model. @@ -265,7 +265,7 @@ def sentence_vectorizer(self, text: str, use_mean: bool = True) -> ndarray: :return: 300-dimension vector representing the given sentence in form of :mod:`numpy` array - :rtype: :class:`numpy.ndarray((1,300))` + :rtype: numpy.ndarray :Example: diff --git a/pythainlp/wsd/core.py b/pythainlp/wsd/core.py index 86870d67e..9682e4930 100644 --- a/pythainlp/wsd/core.py +++ b/pythainlp/wsd/core.py @@ -66,7 +66,7 @@ def get_sense( sentence. :return: a list of definitions and distances (1 - cos_sim) or \ an empty list (if word is not in the dictionary) - :rtype: List[Tuple[str, float]] + :rtype: list[tuple[str, float]] We get the ideas from `Context-Aware Semantic Similarity Measurement for \ Unsupervised Word Sense Disambiguation \